{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Off-Policy VPG with Baseline to Play CartPole-v0\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import imp\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras\n",
    "from tensorflow import nn\n",
    "from tensorflow import optimizers\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import losses\n",
    "\n",
    "imp.reload(logging)\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:32:33 [INFO] env: <CartPoleEnv<CartPole-v0>>\n",
      "22:32:33 [INFO] action_space: Discrete(2)\n",
      "22:32:33 [INFO] observation_space: Box(-3.4028234663852886e+38, 3.4028234663852886e+38, (4,), float32)\n",
      "22:32:33 [INFO] reward_range: (-inf, inf)\n",
      "22:32:33 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}\n",
      "22:32:33 [INFO] _max_episode_steps: 200\n",
      "22:32:33 [INFO] _elapsed_steps: None\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OffPolicyVPGwBaselineAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        def dot(y_true, y_pred):\n",
    "            return -tf.reduce_sum(y_true * y_pred, axis=-1)\n",
    "\n",
    "        self.policy_net = self.build_net(hidden_sizes=[],\n",
    "                output_size=self.action_n,\n",
    "                output_activation=nn.softmax,\n",
    "                loss=dot, learning_rate=0.06)\n",
    "        self.baseline_net = self.build_net(hidden_sizes=[],\n",
    "                learning_rate=0.1)\n",
    "    \n",
    "    def build_net(self, hidden_sizes, output_size=1,\n",
    "            activation=nn.relu, output_activation=None,\n",
    "            use_bias=False, loss=losses.mse, learning_rate=0.001):\n",
    "        model = keras.Sequential()\n",
    "        for hidden_size in hidden_sizes:\n",
    "            model.add(layers.Dense(units=hidden_size,\n",
    "                    activation=activation, use_bias=use_bias))\n",
    "        model.add(layers.Dense(units=output_size,\n",
    "                activation=output_activation, use_bias=use_bias))\n",
    "        optimizer = optimizers.Adam(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "      \n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':            \n",
    "            self.trajectory = []\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if self.mode == 'train':\n",
    "            action = np.random.choice(self.action_n) # use random policy\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "        else:\n",
    "            probs = self.policy_net.predict(observation[np.newaxis])[0]\n",
    "            action = np.random.choice(self.action_n, p=probs)\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        if self.mode == 'train':            \n",
    "            self.learn()\n",
    "\n",
    "    def learn(self):\n",
    "        df = pd.DataFrame(np.array(self.trajectory, dtype=object).reshape(-1, 4),\n",
    "                columns=['state', 'reward', 'done', 'action'])\n",
    "\n",
    "        # train baseline\n",
    "        df['discount'] = self.gamma ** df.index.to_series()\n",
    "        df['discounted_reward'] = df['discount'] * df['reward'].astype(float)\n",
    "        df['discounted_return'] = df['discounted_reward'][::-1].cumsum()\n",
    "        df['return'] = df['discounted_return'] / df['discount']\n",
    "        states = np.stack(df['state'])\n",
    "        returns = df[['return',]].values\n",
    "        self.baseline_net.fit(states, returns, verbose=0)\n",
    "\n",
    "        # train policy\n",
    "        states = np.stack(df['state'])\n",
    "        df['baseline'] = self.baseline_net.predict(states)\n",
    "        df['psi'] = df['discounted_return'] - df['baseline'] * df['discount']\n",
    "        df['behavior_prob'] = 1. / self.action_n\n",
    "        df['sample_weight'] = df['psi'] / df['behavior_prob']\n",
    "        actions = np.eye(self.action_n)[df['action'].astype(int)]\n",
    "        sample_weight = df[['sample_weight',]].values\n",
    "        self.policy_net.fit(states, actions, sample_weight=sample_weight, verbose=0)\n",
    "\n",
    "\n",
    "agent = OffPolicyVPGwBaselineAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:32:35 [INFO] ==== train & verify ====\n",
      "22:32:35 [INFO] NumExpr defaulting to 8 threads.\n",
      "22:32:58 [DEBUG] verify episode 0: reward = 119.00, steps = 119\n",
      "22:33:04 [DEBUG] verify episode 1: reward = 31.00, steps = 31\n",
      "22:33:13 [DEBUG] verify episode 2: reward = 60.00, steps = 60\n",
      "22:33:20 [DEBUG] verify episode 3: reward = 40.00, steps = 40\n",
      "22:33:27 [DEBUG] verify episode 4: reward = 42.00, steps = 42\n",
      "22:33:30 [DEBUG] verify episode 5: reward = 15.00, steps = 15\n",
      "22:33:35 [DEBUG] verify episode 6: reward = 22.00, steps = 22\n",
      "22:33:42 [DEBUG] verify episode 7: reward = 38.00, steps = 38\n",
      "22:33:50 [DEBUG] verify episode 8: reward = 44.00, steps = 44\n",
      "22:34:00 [DEBUG] verify episode 9: reward = 62.00, steps = 62\n",
      "22:34:19 [DEBUG] verify episode 10: reward = 102.00, steps = 102\n",
      "22:34:22 [DEBUG] verify episode 11: reward = 16.00, steps = 16\n",
      "22:34:41 [DEBUG] verify episode 12: reward = 134.00, steps = 134\n",
      "22:34:51 [DEBUG] verify episode 13: reward = 59.00, steps = 59\n",
      "22:35:07 [DEBUG] verify episode 14: reward = 111.00, steps = 111\n",
      "22:35:09 [DEBUG] verify episode 15: reward = 16.00, steps = 16\n",
      "22:35:13 [DEBUG] verify episode 16: reward = 25.00, steps = 25\n",
      "22:35:26 [DEBUG] verify episode 17: reward = 88.00, steps = 88\n",
      "22:35:36 [DEBUG] verify episode 18: reward = 79.00, steps = 79\n",
      "22:35:46 [DEBUG] verify episode 19: reward = 59.00, steps = 59\n",
      "22:35:54 [DEBUG] verify episode 20: reward = 61.00, steps = 61\n",
      "22:36:06 [DEBUG] verify episode 21: reward = 82.00, steps = 82\n",
      "22:36:11 [DEBUG] verify episode 22: reward = 33.00, steps = 33\n",
      "22:36:22 [DEBUG] verify episode 23: reward = 76.00, steps = 76\n",
      "22:36:27 [DEBUG] verify episode 24: reward = 32.00, steps = 32\n",
      "22:36:35 [DEBUG] verify episode 25: reward = 58.00, steps = 58\n",
      "22:36:44 [DEBUG] verify episode 26: reward = 73.00, steps = 73\n",
      "22:36:54 [DEBUG] verify episode 27: reward = 75.00, steps = 75\n",
      "22:36:59 [DEBUG] verify episode 28: reward = 38.00, steps = 38\n",
      "22:37:05 [DEBUG] verify episode 29: reward = 50.00, steps = 50\n",
      "22:37:15 [DEBUG] verify episode 30: reward = 70.00, steps = 70\n",
      "22:37:20 [DEBUG] verify episode 31: reward = 38.00, steps = 38\n",
      "22:37:30 [DEBUG] verify episode 32: reward = 71.00, steps = 71\n",
      "22:37:40 [DEBUG] verify episode 33: reward = 63.00, steps = 63\n",
      "22:37:45 [DEBUG] verify episode 34: reward = 29.00, steps = 29\n",
      "22:37:53 [DEBUG] verify episode 35: reward = 41.00, steps = 41\n",
      "22:38:05 [DEBUG] verify episode 36: reward = 69.00, steps = 69\n",
      "22:38:21 [DEBUG] verify episode 37: reward = 96.00, steps = 96\n",
      "22:38:36 [DEBUG] verify episode 38: reward = 93.00, steps = 93\n",
      "22:38:49 [DEBUG] verify episode 39: reward = 100.00, steps = 100\n",
      "22:39:00 [DEBUG] verify episode 40: reward = 76.00, steps = 76\n",
      "22:39:08 [DEBUG] verify episode 41: reward = 47.00, steps = 47\n",
      "22:39:17 [DEBUG] verify episode 42: reward = 58.00, steps = 58\n",
      "22:39:21 [DEBUG] verify episode 43: reward = 31.00, steps = 31\n",
      "22:39:29 [DEBUG] verify episode 44: reward = 57.00, steps = 57\n",
      "22:39:35 [DEBUG] verify episode 45: reward = 41.00, steps = 41\n",
      "22:39:49 [DEBUG] verify episode 46: reward = 100.00, steps = 100\n",
      "22:39:59 [DEBUG] verify episode 47: reward = 74.00, steps = 74\n",
      "22:40:05 [DEBUG] verify episode 48: reward = 43.00, steps = 43\n",
      "22:40:17 [DEBUG] verify episode 49: reward = 86.00, steps = 86\n",
      "22:40:24 [DEBUG] verify episode 50: reward = 49.00, steps = 49\n",
      "22:40:30 [DEBUG] verify episode 51: reward = 40.00, steps = 40\n",
      "22:40:39 [DEBUG] verify episode 52: reward = 69.00, steps = 69\n",
      "22:40:50 [DEBUG] verify episode 53: reward = 73.00, steps = 73\n",
      "22:40:56 [DEBUG] verify episode 54: reward = 43.00, steps = 43\n",
      "22:41:07 [DEBUG] verify episode 55: reward = 75.00, steps = 75\n",
      "22:41:13 [DEBUG] verify episode 56: reward = 46.00, steps = 46\n",
      "22:41:22 [DEBUG] verify episode 57: reward = 57.00, steps = 57\n",
      "22:41:27 [DEBUG] verify episode 58: reward = 38.00, steps = 38\n",
      "22:41:34 [DEBUG] verify episode 59: reward = 54.00, steps = 54\n",
      "22:41:42 [DEBUG] verify episode 60: reward = 54.00, steps = 54\n",
      "22:41:50 [DEBUG] verify episode 61: reward = 61.00, steps = 61\n",
      "22:41:58 [DEBUG] verify episode 62: reward = 61.00, steps = 61\n",
      "22:42:03 [DEBUG] verify episode 63: reward = 30.00, steps = 30\n",
      "22:42:10 [DEBUG] verify episode 64: reward = 65.00, steps = 65\n",
      "22:42:19 [DEBUG] verify episode 65: reward = 69.00, steps = 69\n",
      "22:42:32 [DEBUG] verify episode 66: reward = 104.00, steps = 104\n",
      "22:42:37 [DEBUG] verify episode 67: reward = 41.00, steps = 41\n",
      "22:42:52 [DEBUG] verify episode 68: reward = 114.00, steps = 114\n",
      "22:42:59 [DEBUG] verify episode 69: reward = 56.00, steps = 56\n",
      "22:43:03 [DEBUG] verify episode 70: reward = 34.00, steps = 34\n",
      "22:43:12 [DEBUG] verify episode 71: reward = 68.00, steps = 68\n",
      "22:43:21 [DEBUG] verify episode 72: reward = 74.00, steps = 74\n",
      "22:43:26 [DEBUG] verify episode 73: reward = 39.00, steps = 39\n",
      "22:43:36 [DEBUG] verify episode 74: reward = 80.00, steps = 80\n",
      "22:43:45 [DEBUG] verify episode 75: reward = 68.00, steps = 68\n",
      "22:43:55 [DEBUG] verify episode 76: reward = 83.00, steps = 83\n",
      "22:44:06 [DEBUG] verify episode 77: reward = 87.00, steps = 87\n",
      "22:44:16 [DEBUG] verify episode 78: reward = 78.00, steps = 78\n",
      "22:44:21 [DEBUG] verify episode 79: reward = 39.00, steps = 39\n",
      "22:44:28 [DEBUG] verify episode 80: reward = 54.00, steps = 54\n",
      "22:44:35 [DEBUG] verify episode 81: reward = 56.00, steps = 56\n",
      "22:44:42 [DEBUG] verify episode 82: reward = 50.00, steps = 50\n",
      "22:44:54 [DEBUG] verify episode 83: reward = 93.00, steps = 93\n",
      "22:45:01 [DEBUG] verify episode 84: reward = 55.00, steps = 55\n",
      "22:45:08 [DEBUG] verify episode 85: reward = 54.00, steps = 54\n",
      "22:45:13 [DEBUG] verify episode 86: reward = 41.00, steps = 41\n",
      "22:45:21 [DEBUG] verify episode 87: reward = 61.00, steps = 61\n",
      "22:45:32 [DEBUG] verify episode 88: reward = 88.00, steps = 88\n",
      "22:45:43 [DEBUG] verify episode 89: reward = 84.00, steps = 84\n",
      "22:45:50 [DEBUG] verify episode 90: reward = 51.00, steps = 51\n",
      "22:45:58 [DEBUG] verify episode 91: reward = 61.00, steps = 61\n",
      "22:46:05 [DEBUG] verify episode 92: reward = 55.00, steps = 55\n",
      "22:46:14 [DEBUG] verify episode 93: reward = 65.00, steps = 65\n",
      "22:46:22 [DEBUG] verify episode 94: reward = 56.00, steps = 56\n",
      "22:46:35 [DEBUG] verify episode 95: reward = 86.00, steps = 86\n",
      "22:46:41 [DEBUG] verify episode 96: reward = 34.00, steps = 34\n",
      "22:46:48 [DEBUG] verify episode 97: reward = 50.00, steps = 50\n",
      "22:47:00 [DEBUG] verify episode 98: reward = 92.00, steps = 92\n",
      "22:47:08 [DEBUG] verify episode 99: reward = 63.00, steps = 63\n",
      "22:47:21 [DEBUG] verify episode 100: reward = 97.00, steps = 97\n",
      "22:47:31 [DEBUG] verify episode 101: reward = 71.00, steps = 71\n",
      "22:47:41 [DEBUG] verify episode 102: reward = 64.00, steps = 64\n",
      "22:47:53 [DEBUG] verify episode 103: reward = 84.00, steps = 84\n",
      "22:48:00 [DEBUG] verify episode 104: reward = 54.00, steps = 54\n",
      "22:48:06 [DEBUG] verify episode 105: reward = 43.00, steps = 43\n",
      "22:48:24 [DEBUG] verify episode 106: reward = 144.00, steps = 144\n",
      "22:48:33 [DEBUG] verify episode 107: reward = 58.00, steps = 58\n",
      "22:48:44 [DEBUG] verify episode 108: reward = 79.00, steps = 79\n",
      "22:48:51 [DEBUG] verify episode 109: reward = 48.00, steps = 48\n",
      "22:48:59 [DEBUG] verify episode 110: reward = 56.00, steps = 56\n",
      "22:49:11 [DEBUG] verify episode 111: reward = 86.00, steps = 86\n",
      "22:49:23 [DEBUG] verify episode 112: reward = 92.00, steps = 92\n",
      "22:49:33 [DEBUG] verify episode 113: reward = 73.00, steps = 73\n",
      "22:49:50 [DEBUG] verify episode 114: reward = 126.00, steps = 126\n",
      "22:49:59 [DEBUG] verify episode 115: reward = 64.00, steps = 64\n",
      "22:50:10 [DEBUG] verify episode 116: reward = 77.00, steps = 77\n",
      "22:50:25 [DEBUG] verify episode 117: reward = 91.00, steps = 91\n",
      "22:50:34 [DEBUG] verify episode 118: reward = 66.00, steps = 66\n",
      "22:50:49 [DEBUG] verify episode 119: reward = 98.00, steps = 98\n",
      "22:50:59 [DEBUG] verify episode 120: reward = 48.00, steps = 48\n",
      "22:51:11 [DEBUG] verify episode 121: reward = 72.00, steps = 72\n",
      "22:51:21 [DEBUG] verify episode 122: reward = 56.00, steps = 56\n",
      "22:51:34 [DEBUG] verify episode 123: reward = 79.00, steps = 79\n",
      "22:51:44 [DEBUG] verify episode 124: reward = 62.00, steps = 62\n",
      "22:51:54 [DEBUG] verify episode 125: reward = 66.00, steps = 66\n",
      "22:52:07 [DEBUG] verify episode 126: reward = 106.00, steps = 106\n",
      "22:52:17 [DEBUG] verify episode 127: reward = 73.00, steps = 73\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:52:25 [DEBUG] verify episode 128: reward = 58.00, steps = 58\n",
      "22:52:32 [DEBUG] verify episode 129: reward = 45.00, steps = 45\n",
      "22:52:38 [DEBUG] verify episode 130: reward = 42.00, steps = 42\n",
      "22:52:51 [DEBUG] verify episode 131: reward = 96.00, steps = 96\n",
      "22:53:00 [DEBUG] verify episode 132: reward = 66.00, steps = 66\n",
      "22:53:07 [DEBUG] verify episode 133: reward = 49.00, steps = 49\n",
      "22:53:22 [DEBUG] verify episode 134: reward = 107.00, steps = 107\n",
      "22:53:28 [DEBUG] verify episode 135: reward = 42.00, steps = 42\n",
      "22:53:36 [DEBUG] verify episode 136: reward = 55.00, steps = 55\n",
      "22:53:48 [DEBUG] verify episode 137: reward = 75.00, steps = 75\n",
      "22:54:01 [DEBUG] verify episode 138: reward = 70.00, steps = 70\n",
      "22:54:13 [DEBUG] verify episode 139: reward = 59.00, steps = 59\n",
      "22:54:20 [DEBUG] verify episode 140: reward = 35.00, steps = 35\n",
      "22:54:41 [DEBUG] verify episode 141: reward = 105.00, steps = 105\n",
      "22:54:53 [DEBUG] verify episode 142: reward = 65.00, steps = 65\n",
      "22:55:02 [DEBUG] verify episode 143: reward = 48.00, steps = 48\n",
      "22:55:14 [DEBUG] verify episode 144: reward = 62.00, steps = 62\n",
      "22:55:37 [DEBUG] verify episode 145: reward = 109.00, steps = 109\n",
      "22:55:49 [DEBUG] verify episode 146: reward = 64.00, steps = 64\n",
      "22:56:09 [DEBUG] verify episode 147: reward = 102.00, steps = 102\n",
      "22:56:33 [DEBUG] verify episode 148: reward = 113.00, steps = 113\n",
      "22:56:46 [DEBUG] verify episode 149: reward = 64.00, steps = 64\n",
      "22:56:58 [DEBUG] verify episode 150: reward = 52.00, steps = 52\n",
      "22:57:13 [DEBUG] verify episode 151: reward = 68.00, steps = 68\n",
      "22:57:24 [DEBUG] verify episode 152: reward = 49.00, steps = 49\n",
      "22:57:41 [DEBUG] verify episode 153: reward = 88.00, steps = 88\n",
      "22:57:51 [DEBUG] verify episode 154: reward = 46.00, steps = 46\n",
      "22:58:06 [DEBUG] verify episode 155: reward = 82.00, steps = 82\n",
      "22:58:29 [DEBUG] verify episode 156: reward = 130.00, steps = 130\n",
      "22:58:41 [DEBUG] verify episode 157: reward = 63.00, steps = 63\n",
      "22:58:56 [DEBUG] verify episode 158: reward = 78.00, steps = 78\n",
      "22:59:05 [DEBUG] verify episode 159: reward = 57.00, steps = 57\n",
      "22:59:17 [DEBUG] verify episode 160: reward = 77.00, steps = 77\n",
      "22:59:30 [DEBUG] verify episode 161: reward = 77.00, steps = 77\n",
      "22:59:38 [DEBUG] verify episode 162: reward = 51.00, steps = 51\n",
      "22:59:58 [DEBUG] verify episode 163: reward = 123.00, steps = 123\n",
      "23:00:24 [DEBUG] verify episode 164: reward = 177.00, steps = 177\n",
      "23:00:38 [DEBUG] verify episode 165: reward = 90.00, steps = 90\n",
      "23:00:46 [DEBUG] verify episode 166: reward = 54.00, steps = 54\n",
      "23:00:55 [DEBUG] verify episode 167: reward = 56.00, steps = 56\n",
      "23:01:02 [DEBUG] verify episode 168: reward = 42.00, steps = 42\n",
      "23:01:13 [DEBUG] verify episode 169: reward = 78.00, steps = 78\n",
      "23:01:21 [DEBUG] verify episode 170: reward = 63.00, steps = 63\n",
      "23:01:32 [DEBUG] verify episode 171: reward = 86.00, steps = 86\n",
      "23:01:41 [DEBUG] verify episode 172: reward = 66.00, steps = 66\n",
      "23:01:48 [DEBUG] verify episode 173: reward = 59.00, steps = 59\n",
      "23:01:56 [DEBUG] verify episode 174: reward = 55.00, steps = 55\n",
      "23:02:04 [DEBUG] verify episode 175: reward = 58.00, steps = 58\n",
      "23:02:14 [DEBUG] verify episode 176: reward = 80.00, steps = 80\n",
      "23:02:30 [DEBUG] verify episode 177: reward = 120.00, steps = 120\n",
      "23:02:38 [DEBUG] verify episode 178: reward = 56.00, steps = 56\n",
      "23:02:46 [DEBUG] verify episode 179: reward = 59.00, steps = 59\n",
      "23:03:03 [DEBUG] verify episode 180: reward = 130.00, steps = 130\n",
      "23:03:14 [DEBUG] verify episode 181: reward = 92.00, steps = 92\n",
      "23:03:22 [DEBUG] verify episode 182: reward = 61.00, steps = 61\n",
      "23:03:31 [DEBUG] verify episode 183: reward = 64.00, steps = 64\n",
      "23:03:40 [DEBUG] verify episode 184: reward = 70.00, steps = 70\n",
      "23:03:50 [DEBUG] verify episode 185: reward = 76.00, steps = 76\n",
      "23:03:56 [DEBUG] verify episode 186: reward = 49.00, steps = 49\n",
      "23:04:07 [DEBUG] verify episode 187: reward = 82.00, steps = 82\n",
      "23:04:21 [DEBUG] verify episode 188: reward = 118.00, steps = 118\n",
      "23:04:30 [DEBUG] verify episode 189: reward = 69.00, steps = 69\n",
      "23:04:41 [DEBUG] verify episode 190: reward = 91.00, steps = 91\n",
      "23:04:48 [DEBUG] verify episode 191: reward = 59.00, steps = 59\n",
      "23:04:59 [DEBUG] verify episode 192: reward = 88.00, steps = 88\n",
      "23:05:10 [DEBUG] verify episode 193: reward = 88.00, steps = 88\n",
      "23:05:23 [DEBUG] verify episode 194: reward = 107.00, steps = 107\n",
      "23:05:32 [DEBUG] verify episode 195: reward = 72.00, steps = 72\n",
      "23:05:41 [DEBUG] verify episode 196: reward = 76.00, steps = 76\n",
      "23:05:51 [DEBUG] verify episode 197: reward = 78.00, steps = 78\n",
      "23:05:58 [DEBUG] verify episode 198: reward = 57.00, steps = 57\n",
      "23:06:05 [DEBUG] verify episode 199: reward = 60.00, steps = 60\n",
      "23:06:11 [DEBUG] verify episode 200: reward = 50.00, steps = 50\n",
      "23:06:19 [DEBUG] verify episode 201: reward = 65.00, steps = 65\n",
      "23:06:24 [DEBUG] verify episode 202: reward = 39.00, steps = 39\n",
      "23:06:33 [DEBUG] verify episode 203: reward = 68.00, steps = 68\n",
      "23:06:44 [DEBUG] verify episode 204: reward = 98.00, steps = 98\n",
      "23:06:53 [DEBUG] verify episode 205: reward = 69.00, steps = 69\n",
      "23:07:11 [DEBUG] verify episode 206: reward = 148.00, steps = 148\n",
      "23:07:18 [DEBUG] verify episode 207: reward = 63.00, steps = 63\n",
      "23:07:28 [DEBUG] verify episode 208: reward = 79.00, steps = 79\n",
      "23:07:33 [DEBUG] verify episode 209: reward = 42.00, steps = 42\n",
      "23:07:40 [DEBUG] verify episode 210: reward = 54.00, steps = 54\n",
      "23:07:47 [DEBUG] verify episode 211: reward = 57.00, steps = 57\n",
      "23:08:04 [DEBUG] verify episode 212: reward = 142.00, steps = 142\n",
      "23:08:11 [DEBUG] verify episode 213: reward = 55.00, steps = 55\n",
      "23:08:24 [DEBUG] verify episode 214: reward = 109.00, steps = 109\n",
      "23:08:38 [DEBUG] verify episode 215: reward = 114.00, steps = 114\n",
      "23:08:45 [DEBUG] verify episode 216: reward = 57.00, steps = 57\n",
      "23:08:58 [DEBUG] verify episode 217: reward = 110.00, steps = 110\n",
      "23:09:07 [DEBUG] verify episode 218: reward = 68.00, steps = 68\n",
      "23:09:17 [DEBUG] verify episode 219: reward = 89.00, steps = 89\n",
      "23:09:28 [DEBUG] verify episode 220: reward = 77.00, steps = 77\n",
      "23:09:37 [DEBUG] verify episode 221: reward = 62.00, steps = 62\n",
      "23:09:46 [DEBUG] verify episode 222: reward = 54.00, steps = 54\n",
      "23:09:53 [DEBUG] verify episode 223: reward = 53.00, steps = 53\n",
      "23:10:03 [DEBUG] verify episode 224: reward = 69.00, steps = 69\n",
      "23:10:13 [DEBUG] verify episode 225: reward = 67.00, steps = 67\n",
      "23:10:22 [DEBUG] verify episode 226: reward = 57.00, steps = 57\n",
      "23:10:36 [DEBUG] verify episode 227: reward = 100.00, steps = 100\n",
      "23:10:47 [DEBUG] verify episode 228: reward = 69.00, steps = 69\n",
      "23:11:00 [DEBUG] verify episode 229: reward = 95.00, steps = 95\n",
      "23:11:12 [DEBUG] verify episode 230: reward = 74.00, steps = 74\n",
      "23:11:34 [DEBUG] verify episode 231: reward = 144.00, steps = 144\n",
      "23:11:43 [DEBUG] verify episode 232: reward = 70.00, steps = 70\n",
      "23:11:53 [DEBUG] verify episode 233: reward = 81.00, steps = 81\n",
      "23:12:04 [DEBUG] verify episode 234: reward = 79.00, steps = 79\n",
      "23:12:16 [DEBUG] verify episode 235: reward = 87.00, steps = 87\n",
      "23:12:30 [DEBUG] verify episode 236: reward = 110.00, steps = 110\n",
      "23:12:35 [DEBUG] verify episode 237: reward = 47.00, steps = 47\n",
      "23:12:45 [DEBUG] verify episode 238: reward = 77.00, steps = 77\n",
      "23:12:54 [DEBUG] verify episode 239: reward = 74.00, steps = 74\n",
      "23:13:07 [DEBUG] verify episode 240: reward = 103.00, steps = 103\n",
      "23:13:15 [DEBUG] verify episode 241: reward = 64.00, steps = 64\n",
      "23:13:24 [DEBUG] verify episode 242: reward = 70.00, steps = 70\n",
      "23:13:31 [DEBUG] verify episode 243: reward = 57.00, steps = 57\n",
      "23:13:43 [DEBUG] verify episode 244: reward = 97.00, steps = 97\n",
      "23:13:54 [DEBUG] verify episode 245: reward = 91.00, steps = 91\n",
      "23:14:07 [DEBUG] verify episode 246: reward = 104.00, steps = 104\n",
      "23:14:21 [DEBUG] verify episode 247: reward = 117.00, steps = 117\n",
      "23:14:28 [DEBUG] verify episode 248: reward = 55.00, steps = 55\n",
      "23:14:36 [DEBUG] verify episode 249: reward = 69.00, steps = 69\n",
      "23:14:52 [DEBUG] verify episode 250: reward = 131.00, steps = 131\n",
      "23:15:06 [DEBUG] verify episode 251: reward = 115.00, steps = 115\n",
      "23:15:14 [DEBUG] verify episode 252: reward = 62.00, steps = 62\n",
      "23:15:24 [DEBUG] verify episode 253: reward = 88.00, steps = 88\n",
      "23:15:37 [DEBUG] verify episode 254: reward = 108.00, steps = 108\n",
      "23:15:52 [DEBUG] verify episode 255: reward = 114.00, steps = 114\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23:16:01 [DEBUG] verify episode 256: reward = 74.00, steps = 74\n",
      "23:16:14 [DEBUG] verify episode 257: reward = 111.00, steps = 111\n",
      "23:16:28 [DEBUG] verify episode 258: reward = 118.00, steps = 118\n",
      "23:16:44 [DEBUG] verify episode 259: reward = 124.00, steps = 124\n",
      "23:16:50 [DEBUG] verify episode 260: reward = 47.00, steps = 47\n",
      "23:17:04 [DEBUG] verify episode 261: reward = 116.00, steps = 116\n",
      "23:17:12 [DEBUG] verify episode 262: reward = 68.00, steps = 68\n",
      "23:17:19 [DEBUG] verify episode 263: reward = 60.00, steps = 60\n",
      "23:17:32 [DEBUG] verify episode 264: reward = 105.00, steps = 105\n",
      "23:17:39 [DEBUG] verify episode 265: reward = 51.00, steps = 51\n",
      "23:17:44 [DEBUG] verify episode 266: reward = 46.00, steps = 46\n",
      "23:17:57 [DEBUG] verify episode 267: reward = 108.00, steps = 108\n",
      "23:18:05 [DEBUG] verify episode 268: reward = 65.00, steps = 65\n",
      "23:18:11 [DEBUG] verify episode 269: reward = 43.00, steps = 43\n",
      "23:18:23 [DEBUG] verify episode 270: reward = 105.00, steps = 105\n",
      "23:18:31 [DEBUG] verify episode 271: reward = 61.00, steps = 61\n",
      "23:18:41 [DEBUG] verify episode 272: reward = 81.00, steps = 81\n",
      "23:18:50 [DEBUG] verify episode 273: reward = 73.00, steps = 73\n",
      "23:19:06 [DEBUG] verify episode 274: reward = 112.00, steps = 112\n",
      "23:19:18 [DEBUG] verify episode 275: reward = 85.00, steps = 85\n",
      "23:19:30 [DEBUG] verify episode 276: reward = 88.00, steps = 88\n",
      "23:19:41 [DEBUG] verify episode 277: reward = 76.00, steps = 76\n",
      "23:19:50 [DEBUG] verify episode 278: reward = 71.00, steps = 71\n",
      "23:19:58 [DEBUG] verify episode 279: reward = 51.00, steps = 51\n",
      "23:20:05 [DEBUG] verify episode 280: reward = 55.00, steps = 55\n",
      "23:20:14 [DEBUG] verify episode 281: reward = 61.00, steps = 61\n",
      "23:20:21 [DEBUG] verify episode 282: reward = 55.00, steps = 55\n",
      "23:20:31 [DEBUG] verify episode 283: reward = 75.00, steps = 75\n",
      "23:20:38 [DEBUG] verify episode 284: reward = 59.00, steps = 59\n",
      "23:20:46 [DEBUG] verify episode 285: reward = 64.00, steps = 64\n",
      "23:20:54 [DEBUG] verify episode 286: reward = 67.00, steps = 67\n",
      "23:21:04 [DEBUG] verify episode 287: reward = 80.00, steps = 80\n",
      "23:21:13 [DEBUG] verify episode 288: reward = 69.00, steps = 69\n",
      "23:21:30 [DEBUG] verify episode 289: reward = 140.00, steps = 140\n",
      "23:21:42 [DEBUG] verify episode 290: reward = 90.00, steps = 90\n",
      "23:21:55 [DEBUG] verify episode 291: reward = 110.00, steps = 110\n",
      "23:22:10 [DEBUG] verify episode 292: reward = 109.00, steps = 109\n",
      "23:22:20 [DEBUG] verify episode 293: reward = 82.00, steps = 82\n",
      "23:22:29 [DEBUG] verify episode 294: reward = 60.00, steps = 60\n",
      "23:22:40 [DEBUG] verify episode 295: reward = 76.00, steps = 76\n",
      "23:22:56 [DEBUG] verify episode 296: reward = 102.00, steps = 102\n",
      "23:23:04 [DEBUG] verify episode 297: reward = 57.00, steps = 57\n",
      "23:23:12 [DEBUG] verify episode 298: reward = 63.00, steps = 63\n",
      "23:23:24 [DEBUG] verify episode 299: reward = 89.00, steps = 89\n",
      "23:23:32 [DEBUG] verify episode 300: reward = 56.00, steps = 56\n",
      "23:23:41 [DEBUG] verify episode 301: reward = 68.00, steps = 68\n",
      "23:23:52 [DEBUG] verify episode 302: reward = 85.00, steps = 85\n",
      "23:24:03 [DEBUG] verify episode 303: reward = 83.00, steps = 83\n",
      "23:24:14 [DEBUG] verify episode 304: reward = 78.00, steps = 78\n",
      "23:24:25 [DEBUG] verify episode 305: reward = 81.00, steps = 81\n",
      "23:24:32 [DEBUG] verify episode 306: reward = 55.00, steps = 55\n",
      "23:24:45 [DEBUG] verify episode 307: reward = 95.00, steps = 95\n",
      "23:24:53 [DEBUG] verify episode 308: reward = 57.00, steps = 57\n",
      "23:25:03 [DEBUG] verify episode 309: reward = 77.00, steps = 77\n",
      "23:25:14 [DEBUG] verify episode 310: reward = 92.00, steps = 92\n",
      "23:25:24 [DEBUG] verify episode 311: reward = 69.00, steps = 69\n",
      "23:25:45 [DEBUG] verify episode 312: reward = 169.00, steps = 169\n",
      "23:25:52 [DEBUG] verify episode 313: reward = 56.00, steps = 56\n",
      "23:26:04 [DEBUG] verify episode 314: reward = 85.00, steps = 85\n",
      "23:26:14 [DEBUG] verify episode 315: reward = 76.00, steps = 76\n",
      "23:26:25 [DEBUG] verify episode 316: reward = 77.00, steps = 77\n",
      "23:26:47 [DEBUG] verify episode 317: reward = 149.00, steps = 149\n",
      "23:27:04 [DEBUG] verify episode 318: reward = 134.00, steps = 134\n",
      "23:27:15 [DEBUG] verify episode 319: reward = 81.00, steps = 81\n",
      "23:27:26 [DEBUG] verify episode 320: reward = 76.00, steps = 76\n",
      "23:27:33 [DEBUG] verify episode 321: reward = 53.00, steps = 53\n",
      "23:27:43 [DEBUG] verify episode 322: reward = 74.00, steps = 74\n",
      "23:28:07 [DEBUG] verify episode 323: reward = 171.00, steps = 171\n",
      "23:28:18 [DEBUG] verify episode 324: reward = 74.00, steps = 74\n",
      "23:28:25 [DEBUG] verify episode 325: reward = 48.00, steps = 48\n",
      "23:28:32 [DEBUG] verify episode 326: reward = 48.00, steps = 48\n",
      "23:28:42 [DEBUG] verify episode 327: reward = 78.00, steps = 78\n",
      "23:28:57 [DEBUG] verify episode 328: reward = 105.00, steps = 105\n",
      "23:29:08 [DEBUG] verify episode 329: reward = 72.00, steps = 72\n",
      "23:29:16 [DEBUG] verify episode 330: reward = 57.00, steps = 57\n",
      "23:29:42 [DEBUG] verify episode 331: reward = 200.00, steps = 200\n",
      "23:29:55 [DEBUG] verify episode 332: reward = 97.00, steps = 97\n",
      "23:30:05 [DEBUG] verify episode 333: reward = 80.00, steps = 80\n",
      "23:30:18 [DEBUG] verify episode 334: reward = 94.00, steps = 94\n",
      "23:30:29 [DEBUG] verify episode 335: reward = 84.00, steps = 84\n",
      "23:30:40 [DEBUG] verify episode 336: reward = 81.00, steps = 81\n",
      "23:30:51 [DEBUG] verify episode 337: reward = 79.00, steps = 79\n",
      "23:30:59 [DEBUG] verify episode 338: reward = 61.00, steps = 61\n",
      "23:31:10 [DEBUG] verify episode 339: reward = 97.00, steps = 97\n",
      "23:31:29 [DEBUG] verify episode 340: reward = 154.00, steps = 154\n",
      "23:31:37 [DEBUG] verify episode 341: reward = 75.00, steps = 75\n",
      "23:31:50 [DEBUG] verify episode 342: reward = 108.00, steps = 108\n",
      "23:32:02 [DEBUG] verify episode 343: reward = 71.00, steps = 71\n",
      "23:32:18 [DEBUG] verify episode 344: reward = 137.00, steps = 137\n",
      "23:32:26 [DEBUG] verify episode 345: reward = 63.00, steps = 63\n",
      "23:32:39 [DEBUG] verify episode 346: reward = 109.00, steps = 109\n",
      "23:32:49 [DEBUG] verify episode 347: reward = 89.00, steps = 89\n",
      "23:33:00 [DEBUG] verify episode 348: reward = 97.00, steps = 97\n",
      "23:33:15 [DEBUG] verify episode 349: reward = 140.00, steps = 140\n",
      "23:33:24 [DEBUG] verify episode 350: reward = 84.00, steps = 84\n",
      "23:33:35 [DEBUG] verify episode 351: reward = 92.00, steps = 92\n",
      "23:33:45 [DEBUG] verify episode 352: reward = 90.00, steps = 90\n",
      "23:33:55 [DEBUG] verify episode 353: reward = 94.00, steps = 94\n",
      "23:34:03 [DEBUG] verify episode 354: reward = 84.00, steps = 84\n",
      "23:34:18 [DEBUG] verify episode 355: reward = 94.00, steps = 94\n",
      "23:34:24 [DEBUG] verify episode 356: reward = 59.00, steps = 59\n",
      "23:34:31 [DEBUG] verify episode 357: reward = 65.00, steps = 65\n",
      "23:34:48 [DEBUG] verify episode 358: reward = 170.00, steps = 170\n",
      "23:34:56 [DEBUG] verify episode 359: reward = 76.00, steps = 76\n",
      "23:35:05 [DEBUG] verify episode 360: reward = 83.00, steps = 83\n",
      "23:35:15 [DEBUG] verify episode 361: reward = 87.00, steps = 87\n",
      "23:35:23 [DEBUG] verify episode 362: reward = 68.00, steps = 68\n",
      "23:35:31 [DEBUG] verify episode 363: reward = 69.00, steps = 69\n",
      "23:35:37 [DEBUG] verify episode 364: reward = 56.00, steps = 56\n",
      "23:35:49 [DEBUG] verify episode 365: reward = 107.00, steps = 107\n",
      "23:36:03 [DEBUG] verify episode 366: reward = 147.00, steps = 147\n",
      "23:36:13 [DEBUG] verify episode 367: reward = 94.00, steps = 94\n",
      "23:36:23 [DEBUG] verify episode 368: reward = 99.00, steps = 99\n",
      "23:36:29 [DEBUG] verify episode 369: reward = 57.00, steps = 57\n",
      "23:36:40 [DEBUG] verify episode 370: reward = 109.00, steps = 109\n",
      "23:36:49 [DEBUG] verify episode 371: reward = 80.00, steps = 80\n",
      "23:36:58 [DEBUG] verify episode 372: reward = 93.00, steps = 93\n",
      "23:37:06 [DEBUG] verify episode 373: reward = 73.00, steps = 73\n",
      "23:37:12 [DEBUG] verify episode 374: reward = 57.00, steps = 57\n",
      "23:37:19 [DEBUG] verify episode 375: reward = 59.00, steps = 59\n",
      "23:37:28 [DEBUG] verify episode 376: reward = 86.00, steps = 86\n",
      "23:37:35 [DEBUG] verify episode 377: reward = 79.00, steps = 79\n",
      "23:37:43 [DEBUG] verify episode 378: reward = 67.00, steps = 67\n",
      "23:37:57 [DEBUG] verify episode 379: reward = 134.00, steps = 134\n",
      "23:38:06 [DEBUG] verify episode 380: reward = 85.00, steps = 85\n",
      "23:38:14 [DEBUG] verify episode 381: reward = 86.00, steps = 86\n",
      "23:38:24 [DEBUG] verify episode 382: reward = 90.00, steps = 90\n",
      "23:38:33 [DEBUG] verify episode 383: reward = 90.00, steps = 90\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23:38:46 [DEBUG] verify episode 384: reward = 122.00, steps = 122\n",
      "23:38:54 [DEBUG] verify episode 385: reward = 77.00, steps = 77\n",
      "23:39:04 [DEBUG] verify episode 386: reward = 91.00, steps = 91\n",
      "23:39:12 [DEBUG] verify episode 387: reward = 71.00, steps = 71\n",
      "23:39:25 [DEBUG] verify episode 388: reward = 115.00, steps = 115\n",
      "23:39:40 [DEBUG] verify episode 389: reward = 133.00, steps = 133\n",
      "23:39:48 [DEBUG] verify episode 390: reward = 72.00, steps = 72\n",
      "23:39:57 [DEBUG] verify episode 391: reward = 81.00, steps = 81\n",
      "23:40:05 [DEBUG] verify episode 392: reward = 68.00, steps = 68\n",
      "23:40:21 [DEBUG] verify episode 393: reward = 142.00, steps = 142\n",
      "23:40:38 [DEBUG] verify episode 394: reward = 154.00, steps = 154\n",
      "23:40:46 [DEBUG] verify episode 395: reward = 70.00, steps = 70\n",
      "23:41:01 [DEBUG] verify episode 396: reward = 129.00, steps = 129\n",
      "23:41:19 [DEBUG] verify episode 397: reward = 162.00, steps = 162\n",
      "23:41:33 [DEBUG] verify episode 398: reward = 134.00, steps = 134\n",
      "23:41:47 [DEBUG] verify episode 399: reward = 135.00, steps = 135\n",
      "23:41:56 [DEBUG] verify episode 400: reward = 83.00, steps = 83\n",
      "23:42:11 [DEBUG] verify episode 401: reward = 142.00, steps = 142\n",
      "23:42:18 [DEBUG] verify episode 402: reward = 72.00, steps = 72\n",
      "23:42:27 [DEBUG] verify episode 403: reward = 78.00, steps = 78\n",
      "23:42:41 [DEBUG] verify episode 404: reward = 139.00, steps = 139\n",
      "23:42:51 [DEBUG] verify episode 405: reward = 90.00, steps = 90\n",
      "23:42:59 [DEBUG] verify episode 406: reward = 76.00, steps = 76\n",
      "23:43:09 [DEBUG] verify episode 407: reward = 95.00, steps = 95\n",
      "23:43:16 [DEBUG] verify episode 408: reward = 56.00, steps = 56\n",
      "23:43:26 [DEBUG] verify episode 409: reward = 85.00, steps = 85\n",
      "23:43:42 [DEBUG] verify episode 410: reward = 141.00, steps = 141\n",
      "23:43:54 [DEBUG] verify episode 411: reward = 89.00, steps = 89\n",
      "23:44:02 [DEBUG] verify episode 412: reward = 67.00, steps = 67\n",
      "23:44:09 [DEBUG] verify episode 413: reward = 66.00, steps = 66\n",
      "23:44:24 [DEBUG] verify episode 414: reward = 115.00, steps = 115\n",
      "23:44:34 [DEBUG] verify episode 415: reward = 86.00, steps = 86\n",
      "23:44:42 [DEBUG] verify episode 416: reward = 73.00, steps = 73\n",
      "23:44:53 [DEBUG] verify episode 417: reward = 85.00, steps = 85\n",
      "23:45:08 [DEBUG] verify episode 418: reward = 102.00, steps = 102\n",
      "23:45:25 [DEBUG] verify episode 419: reward = 106.00, steps = 106\n",
      "23:45:35 [DEBUG] verify episode 420: reward = 93.00, steps = 93\n",
      "23:45:46 [DEBUG] verify episode 421: reward = 88.00, steps = 88\n",
      "23:45:56 [DEBUG] verify episode 422: reward = 92.00, steps = 92\n",
      "23:46:08 [DEBUG] verify episode 423: reward = 89.00, steps = 89\n",
      "23:46:17 [DEBUG] verify episode 424: reward = 82.00, steps = 82\n",
      "23:46:28 [DEBUG] verify episode 425: reward = 89.00, steps = 89\n",
      "23:46:40 [DEBUG] verify episode 426: reward = 103.00, steps = 103\n",
      "23:46:49 [DEBUG] verify episode 427: reward = 74.00, steps = 74\n",
      "23:47:01 [DEBUG] verify episode 428: reward = 85.00, steps = 85\n",
      "23:47:10 [DEBUG] verify episode 429: reward = 82.00, steps = 82\n",
      "23:47:22 [DEBUG] verify episode 430: reward = 105.00, steps = 105\n",
      "23:47:32 [DEBUG] verify episode 431: reward = 90.00, steps = 90\n",
      "23:47:50 [DEBUG] verify episode 432: reward = 156.00, steps = 156\n",
      "23:48:00 [DEBUG] verify episode 433: reward = 83.00, steps = 83\n",
      "23:48:12 [DEBUG] verify episode 434: reward = 102.00, steps = 102\n",
      "23:48:28 [DEBUG] verify episode 435: reward = 124.00, steps = 124\n",
      "23:48:46 [DEBUG] verify episode 436: reward = 161.00, steps = 161\n",
      "23:48:57 [DEBUG] verify episode 437: reward = 98.00, steps = 98\n",
      "23:49:12 [DEBUG] verify episode 438: reward = 120.00, steps = 120\n",
      "23:49:22 [DEBUG] verify episode 439: reward = 82.00, steps = 82\n",
      "23:49:33 [DEBUG] verify episode 440: reward = 85.00, steps = 85\n",
      "23:49:40 [DEBUG] verify episode 441: reward = 65.00, steps = 65\n",
      "23:49:49 [DEBUG] verify episode 442: reward = 70.00, steps = 70\n",
      "23:49:58 [DEBUG] verify episode 443: reward = 70.00, steps = 70\n",
      "23:50:07 [DEBUG] verify episode 444: reward = 74.00, steps = 74\n",
      "23:50:16 [DEBUG] verify episode 445: reward = 88.00, steps = 88\n",
      "23:50:25 [DEBUG] verify episode 446: reward = 59.00, steps = 59\n",
      "23:50:35 [DEBUG] verify episode 447: reward = 88.00, steps = 88\n",
      "23:50:44 [DEBUG] verify episode 448: reward = 82.00, steps = 82\n",
      "23:51:01 [DEBUG] verify episode 449: reward = 135.00, steps = 135\n",
      "23:51:12 [DEBUG] verify episode 450: reward = 98.00, steps = 98\n",
      "23:51:24 [DEBUG] verify episode 451: reward = 92.00, steps = 92\n",
      "23:51:32 [DEBUG] verify episode 452: reward = 72.00, steps = 72\n",
      "23:51:43 [DEBUG] verify episode 453: reward = 92.00, steps = 92\n",
      "23:51:54 [DEBUG] verify episode 454: reward = 99.00, steps = 99\n",
      "23:52:03 [DEBUG] verify episode 455: reward = 79.00, steps = 79\n",
      "23:52:14 [DEBUG] verify episode 456: reward = 103.00, steps = 103\n",
      "23:52:37 [DEBUG] verify episode 457: reward = 200.00, steps = 200\n",
      "23:52:53 [DEBUG] verify episode 458: reward = 135.00, steps = 135\n",
      "23:53:04 [DEBUG] verify episode 459: reward = 104.00, steps = 104\n",
      "23:53:13 [DEBUG] verify episode 460: reward = 81.00, steps = 81\n",
      "23:53:23 [DEBUG] verify episode 461: reward = 104.00, steps = 104\n",
      "23:53:32 [DEBUG] verify episode 462: reward = 89.00, steps = 89\n",
      "23:53:42 [DEBUG] verify episode 463: reward = 100.00, steps = 100\n",
      "23:53:51 [DEBUG] verify episode 464: reward = 81.00, steps = 81\n",
      "23:54:03 [DEBUG] verify episode 465: reward = 118.00, steps = 118\n",
      "23:54:16 [DEBUG] verify episode 466: reward = 133.00, steps = 133\n",
      "23:54:23 [DEBUG] verify episode 467: reward = 66.00, steps = 66\n",
      "23:54:33 [DEBUG] verify episode 468: reward = 105.00, steps = 105\n",
      "23:54:44 [DEBUG] verify episode 469: reward = 102.00, steps = 102\n",
      "23:54:54 [DEBUG] verify episode 470: reward = 102.00, steps = 102\n",
      "23:55:04 [DEBUG] verify episode 471: reward = 97.00, steps = 97\n",
      "23:55:19 [DEBUG] verify episode 472: reward = 145.00, steps = 145\n",
      "23:55:35 [DEBUG] verify episode 473: reward = 160.00, steps = 160\n",
      "23:55:43 [DEBUG] verify episode 474: reward = 81.00, steps = 81\n",
      "23:55:53 [DEBUG] verify episode 475: reward = 102.00, steps = 102\n",
      "23:56:02 [DEBUG] verify episode 476: reward = 79.00, steps = 79\n",
      "23:56:09 [DEBUG] verify episode 477: reward = 70.00, steps = 70\n",
      "23:56:21 [DEBUG] verify episode 478: reward = 89.00, steps = 89\n",
      "23:56:33 [DEBUG] verify episode 479: reward = 112.00, steps = 112\n",
      "23:56:40 [DEBUG] verify episode 480: reward = 74.00, steps = 74\n",
      "23:56:53 [DEBUG] verify episode 481: reward = 123.00, steps = 123\n",
      "23:57:02 [DEBUG] verify episode 482: reward = 88.00, steps = 88\n",
      "23:57:10 [DEBUG] verify episode 483: reward = 75.00, steps = 75\n",
      "23:57:25 [DEBUG] verify episode 484: reward = 153.00, steps = 153\n",
      "23:57:33 [DEBUG] verify episode 485: reward = 70.00, steps = 70\n",
      "23:57:45 [DEBUG] verify episode 486: reward = 126.00, steps = 126\n",
      "23:57:54 [DEBUG] verify episode 487: reward = 85.00, steps = 85\n",
      "23:58:04 [DEBUG] verify episode 488: reward = 96.00, steps = 96\n",
      "23:58:17 [DEBUG] verify episode 489: reward = 133.00, steps = 133\n",
      "23:58:31 [DEBUG] verify episode 490: reward = 140.00, steps = 140\n",
      "23:58:39 [DEBUG] verify episode 491: reward = 76.00, steps = 76\n",
      "23:58:48 [DEBUG] verify episode 492: reward = 87.00, steps = 87\n",
      "23:59:01 [DEBUG] verify episode 493: reward = 124.00, steps = 124\n",
      "23:59:11 [DEBUG] verify episode 494: reward = 100.00, steps = 100\n",
      "23:59:24 [DEBUG] verify episode 495: reward = 124.00, steps = 124\n",
      "23:59:33 [DEBUG] verify episode 496: reward = 89.00, steps = 89\n",
      "23:59:42 [DEBUG] verify episode 497: reward = 81.00, steps = 81\n",
      "23:59:51 [DEBUG] verify episode 498: reward = 88.00, steps = 88\n",
      "00:00:01 [DEBUG] verify episode 499: reward = 98.00, steps = 98\n",
      "00:00:11 [DEBUG] verify episode 500: reward = 100.00, steps = 100\n",
      "00:00:21 [DEBUG] verify episode 501: reward = 99.00, steps = 99\n",
      "00:00:32 [DEBUG] verify episode 502: reward = 107.00, steps = 107\n",
      "00:00:40 [DEBUG] verify episode 503: reward = 76.00, steps = 76\n",
      "00:00:46 [DEBUG] verify episode 504: reward = 59.00, steps = 59\n",
      "00:00:53 [DEBUG] verify episode 505: reward = 66.00, steps = 66\n",
      "00:01:02 [DEBUG] verify episode 506: reward = 89.00, steps = 89\n",
      "00:01:10 [DEBUG] verify episode 507: reward = 70.00, steps = 70\n",
      "00:01:16 [DEBUG] verify episode 508: reward = 62.00, steps = 62\n",
      "00:01:26 [DEBUG] verify episode 509: reward = 92.00, steps = 92\n",
      "00:01:35 [DEBUG] verify episode 510: reward = 88.00, steps = 88\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:01:43 [DEBUG] verify episode 511: reward = 77.00, steps = 77\n",
      "00:01:55 [DEBUG] verify episode 512: reward = 120.00, steps = 120\n",
      "00:02:11 [DEBUG] verify episode 513: reward = 149.00, steps = 149\n",
      "00:02:22 [DEBUG] verify episode 514: reward = 111.00, steps = 111\n",
      "00:02:36 [DEBUG] verify episode 515: reward = 131.00, steps = 131\n",
      "00:02:44 [DEBUG] verify episode 516: reward = 71.00, steps = 71\n",
      "00:02:53 [DEBUG] verify episode 517: reward = 96.00, steps = 96\n",
      "00:03:04 [DEBUG] verify episode 518: reward = 104.00, steps = 104\n",
      "00:03:15 [DEBUG] verify episode 519: reward = 111.00, steps = 111\n",
      "00:03:28 [DEBUG] verify episode 520: reward = 131.00, steps = 131\n",
      "00:03:35 [DEBUG] verify episode 521: reward = 64.00, steps = 64\n",
      "00:03:45 [DEBUG] verify episode 522: reward = 92.00, steps = 92\n",
      "00:03:53 [DEBUG] verify episode 523: reward = 79.00, steps = 79\n",
      "00:04:07 [DEBUG] verify episode 524: reward = 144.00, steps = 144\n",
      "00:04:16 [DEBUG] verify episode 525: reward = 89.00, steps = 89\n",
      "00:04:24 [DEBUG] verify episode 526: reward = 83.00, steps = 83\n",
      "00:04:37 [DEBUG] verify episode 527: reward = 122.00, steps = 122\n",
      "00:04:47 [DEBUG] verify episode 528: reward = 103.00, steps = 103\n",
      "00:04:59 [DEBUG] verify episode 529: reward = 115.00, steps = 115\n",
      "00:05:19 [DEBUG] verify episode 530: reward = 197.00, steps = 197\n",
      "00:05:32 [DEBUG] verify episode 531: reward = 124.00, steps = 124\n",
      "00:05:50 [DEBUG] verify episode 532: reward = 185.00, steps = 185\n",
      "00:06:08 [DEBUG] verify episode 533: reward = 185.00, steps = 185\n",
      "00:06:17 [DEBUG] verify episode 534: reward = 82.00, steps = 82\n",
      "00:06:28 [DEBUG] verify episode 535: reward = 112.00, steps = 112\n",
      "00:06:45 [DEBUG] verify episode 536: reward = 175.00, steps = 175\n",
      "00:07:01 [DEBUG] verify episode 537: reward = 156.00, steps = 156\n",
      "00:07:11 [DEBUG] verify episode 538: reward = 97.00, steps = 97\n",
      "00:07:28 [DEBUG] verify episode 539: reward = 168.00, steps = 168\n",
      "00:07:41 [DEBUG] verify episode 540: reward = 126.00, steps = 126\n",
      "00:07:50 [DEBUG] verify episode 541: reward = 95.00, steps = 95\n",
      "00:08:02 [DEBUG] verify episode 542: reward = 113.00, steps = 113\n",
      "00:08:11 [DEBUG] verify episode 543: reward = 92.00, steps = 92\n",
      "00:08:19 [DEBUG] verify episode 544: reward = 69.00, steps = 69\n",
      "00:08:37 [DEBUG] verify episode 545: reward = 183.00, steps = 183\n",
      "00:08:53 [DEBUG] verify episode 546: reward = 158.00, steps = 158\n",
      "00:09:06 [DEBUG] verify episode 547: reward = 129.00, steps = 129\n",
      "00:09:19 [DEBUG] verify episode 548: reward = 131.00, steps = 131\n",
      "00:09:26 [DEBUG] verify episode 549: reward = 73.00, steps = 73\n",
      "00:09:40 [DEBUG] verify episode 550: reward = 135.00, steps = 135\n",
      "00:09:52 [DEBUG] verify episode 551: reward = 114.00, steps = 114\n",
      "00:10:01 [DEBUG] verify episode 552: reward = 93.00, steps = 93\n",
      "00:10:14 [DEBUG] verify episode 553: reward = 131.00, steps = 131\n",
      "00:10:24 [DEBUG] verify episode 554: reward = 94.00, steps = 94\n",
      "00:10:33 [DEBUG] verify episode 555: reward = 87.00, steps = 87\n",
      "00:10:44 [DEBUG] verify episode 556: reward = 113.00, steps = 113\n",
      "00:10:54 [DEBUG] verify episode 557: reward = 98.00, steps = 98\n",
      "00:11:08 [DEBUG] verify episode 558: reward = 135.00, steps = 135\n",
      "00:11:15 [DEBUG] verify episode 559: reward = 73.00, steps = 73\n",
      "00:11:35 [DEBUG] verify episode 560: reward = 200.00, steps = 200\n",
      "00:11:47 [DEBUG] verify episode 561: reward = 125.00, steps = 125\n",
      "00:12:03 [DEBUG] verify episode 562: reward = 163.00, steps = 163\n",
      "00:12:20 [DEBUG] verify episode 563: reward = 166.00, steps = 166\n",
      "00:12:29 [DEBUG] verify episode 564: reward = 88.00, steps = 88\n",
      "00:12:38 [DEBUG] verify episode 565: reward = 86.00, steps = 86\n",
      "00:12:47 [DEBUG] verify episode 566: reward = 92.00, steps = 92\n",
      "00:13:06 [DEBUG] verify episode 567: reward = 192.00, steps = 192\n",
      "00:13:18 [DEBUG] verify episode 568: reward = 114.00, steps = 114\n",
      "00:13:32 [DEBUG] verify episode 569: reward = 145.00, steps = 145\n",
      "00:13:44 [DEBUG] verify episode 570: reward = 111.00, steps = 111\n",
      "00:13:53 [DEBUG] verify episode 571: reward = 92.00, steps = 92\n",
      "00:14:05 [DEBUG] verify episode 572: reward = 105.00, steps = 105\n",
      "00:14:14 [DEBUG] verify episode 573: reward = 94.00, steps = 94\n",
      "00:14:26 [DEBUG] verify episode 574: reward = 114.00, steps = 114\n",
      "00:14:38 [DEBUG] verify episode 575: reward = 101.00, steps = 101\n",
      "00:14:47 [DEBUG] verify episode 576: reward = 87.00, steps = 87\n",
      "00:14:58 [DEBUG] verify episode 577: reward = 115.00, steps = 115\n",
      "00:15:06 [DEBUG] verify episode 578: reward = 76.00, steps = 76\n",
      "00:15:19 [DEBUG] verify episode 579: reward = 123.00, steps = 123\n",
      "00:15:36 [DEBUG] verify episode 580: reward = 172.00, steps = 172\n",
      "00:15:44 [DEBUG] verify episode 581: reward = 80.00, steps = 80\n",
      "00:16:02 [DEBUG] verify episode 582: reward = 176.00, steps = 176\n",
      "00:16:18 [DEBUG] verify episode 583: reward = 161.00, steps = 161\n",
      "00:16:35 [DEBUG] verify episode 584: reward = 164.00, steps = 164\n",
      "00:16:48 [DEBUG] verify episode 585: reward = 130.00, steps = 130\n",
      "00:17:00 [DEBUG] verify episode 586: reward = 118.00, steps = 118\n",
      "00:17:14 [DEBUG] verify episode 587: reward = 147.00, steps = 147\n",
      "00:17:28 [DEBUG] verify episode 588: reward = 134.00, steps = 134\n",
      "00:17:38 [DEBUG] verify episode 589: reward = 101.00, steps = 101\n",
      "00:17:48 [DEBUG] verify episode 590: reward = 95.00, steps = 95\n",
      "00:17:58 [DEBUG] verify episode 591: reward = 103.00, steps = 103\n",
      "00:18:08 [DEBUG] verify episode 592: reward = 103.00, steps = 103\n",
      "00:18:18 [DEBUG] verify episode 593: reward = 98.00, steps = 98\n",
      "00:18:28 [DEBUG] verify episode 594: reward = 95.00, steps = 95\n",
      "00:18:35 [DEBUG] verify episode 595: reward = 69.00, steps = 69\n",
      "00:18:44 [DEBUG] verify episode 596: reward = 85.00, steps = 85\n",
      "00:18:51 [DEBUG] verify episode 597: reward = 73.00, steps = 73\n",
      "00:18:58 [DEBUG] verify episode 598: reward = 71.00, steps = 71\n",
      "00:19:09 [DEBUG] verify episode 599: reward = 101.00, steps = 101\n",
      "00:19:20 [DEBUG] verify episode 600: reward = 112.00, steps = 112\n",
      "00:19:29 [DEBUG] verify episode 601: reward = 86.00, steps = 86\n",
      "00:19:41 [DEBUG] verify episode 602: reward = 116.00, steps = 116\n",
      "00:19:48 [DEBUG] verify episode 603: reward = 75.00, steps = 75\n",
      "00:20:09 [DEBUG] verify episode 604: reward = 200.00, steps = 200\n",
      "00:20:25 [DEBUG] verify episode 605: reward = 165.00, steps = 165\n",
      "00:20:44 [DEBUG] verify episode 606: reward = 196.00, steps = 196\n",
      "00:20:54 [DEBUG] verify episode 607: reward = 103.00, steps = 103\n",
      "00:21:07 [DEBUG] verify episode 608: reward = 126.00, steps = 126\n",
      "00:21:16 [DEBUG] verify episode 609: reward = 89.00, steps = 89\n",
      "00:21:26 [DEBUG] verify episode 610: reward = 98.00, steps = 98\n",
      "00:21:34 [DEBUG] verify episode 611: reward = 78.00, steps = 78\n",
      "00:21:49 [DEBUG] verify episode 612: reward = 150.00, steps = 150\n",
      "00:21:59 [DEBUG] verify episode 613: reward = 98.00, steps = 98\n",
      "00:22:09 [DEBUG] verify episode 614: reward = 102.00, steps = 102\n",
      "00:22:18 [DEBUG] verify episode 615: reward = 76.00, steps = 76\n",
      "00:22:38 [DEBUG] verify episode 616: reward = 162.00, steps = 162\n",
      "00:22:47 [DEBUG] verify episode 617: reward = 80.00, steps = 80\n",
      "00:23:00 [DEBUG] verify episode 618: reward = 137.00, steps = 137\n",
      "00:23:14 [DEBUG] verify episode 619: reward = 140.00, steps = 140\n",
      "00:23:26 [DEBUG] verify episode 620: reward = 124.00, steps = 124\n",
      "00:23:39 [DEBUG] verify episode 621: reward = 130.00, steps = 130\n",
      "00:23:49 [DEBUG] verify episode 622: reward = 101.00, steps = 101\n",
      "00:24:07 [DEBUG] verify episode 623: reward = 177.00, steps = 177\n",
      "00:24:19 [DEBUG] verify episode 624: reward = 119.00, steps = 119\n",
      "00:24:27 [DEBUG] verify episode 625: reward = 77.00, steps = 77\n",
      "00:24:37 [DEBUG] verify episode 626: reward = 101.00, steps = 101\n",
      "00:24:50 [DEBUG] verify episode 627: reward = 126.00, steps = 126\n",
      "00:25:01 [DEBUG] verify episode 628: reward = 105.00, steps = 105\n",
      "00:25:14 [DEBUG] verify episode 629: reward = 123.00, steps = 123\n",
      "00:25:21 [DEBUG] verify episode 630: reward = 72.00, steps = 72\n",
      "00:25:42 [DEBUG] verify episode 631: reward = 200.00, steps = 200\n",
      "00:25:54 [DEBUG] verify episode 632: reward = 118.00, steps = 118\n",
      "00:26:05 [DEBUG] verify episode 633: reward = 112.00, steps = 112\n",
      "00:26:14 [DEBUG] verify episode 634: reward = 92.00, steps = 92\n",
      "00:26:32 [DEBUG] verify episode 635: reward = 176.00, steps = 176\n",
      "00:26:45 [DEBUG] verify episode 636: reward = 132.00, steps = 132\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:26:55 [DEBUG] verify episode 637: reward = 100.00, steps = 100\n",
      "00:27:08 [DEBUG] verify episode 638: reward = 130.00, steps = 130\n",
      "00:27:17 [DEBUG] verify episode 639: reward = 81.00, steps = 81\n",
      "00:27:29 [DEBUG] verify episode 640: reward = 124.00, steps = 124\n",
      "00:27:39 [DEBUG] verify episode 641: reward = 92.00, steps = 92\n",
      "00:27:52 [DEBUG] verify episode 642: reward = 138.00, steps = 138\n",
      "00:28:05 [DEBUG] verify episode 643: reward = 116.00, steps = 116\n",
      "00:28:14 [DEBUG] verify episode 644: reward = 96.00, steps = 96\n",
      "00:28:23 [DEBUG] verify episode 645: reward = 90.00, steps = 90\n",
      "00:28:38 [DEBUG] verify episode 646: reward = 141.00, steps = 141\n",
      "00:28:46 [DEBUG] verify episode 647: reward = 76.00, steps = 76\n",
      "00:28:58 [DEBUG] verify episode 648: reward = 125.00, steps = 125\n",
      "00:29:08 [DEBUG] verify episode 649: reward = 83.00, steps = 83\n",
      "00:29:19 [DEBUG] verify episode 650: reward = 102.00, steps = 102\n",
      "00:29:33 [DEBUG] verify episode 651: reward = 145.00, steps = 145\n",
      "00:29:40 [DEBUG] verify episode 652: reward = 68.00, steps = 68\n",
      "00:29:52 [DEBUG] verify episode 653: reward = 111.00, steps = 111\n",
      "00:30:04 [DEBUG] verify episode 654: reward = 120.00, steps = 120\n",
      "00:30:15 [DEBUG] verify episode 655: reward = 106.00, steps = 106\n",
      "00:30:26 [DEBUG] verify episode 656: reward = 106.00, steps = 106\n",
      "00:30:36 [DEBUG] verify episode 657: reward = 99.00, steps = 99\n",
      "00:30:56 [DEBUG] verify episode 658: reward = 200.00, steps = 200\n",
      "00:31:05 [DEBUG] verify episode 659: reward = 86.00, steps = 86\n",
      "00:31:17 [DEBUG] verify episode 660: reward = 120.00, steps = 120\n",
      "00:31:27 [DEBUG] verify episode 661: reward = 103.00, steps = 103\n",
      "00:31:40 [DEBUG] verify episode 662: reward = 128.00, steps = 128\n",
      "00:31:52 [DEBUG] verify episode 663: reward = 115.00, steps = 115\n",
      "00:32:01 [DEBUG] verify episode 664: reward = 88.00, steps = 88\n",
      "00:32:11 [DEBUG] verify episode 665: reward = 87.00, steps = 87\n",
      "00:32:23 [DEBUG] verify episode 666: reward = 128.00, steps = 128\n",
      "00:32:33 [DEBUG] verify episode 667: reward = 93.00, steps = 93\n",
      "00:32:44 [DEBUG] verify episode 668: reward = 104.00, steps = 104\n",
      "00:33:02 [DEBUG] verify episode 669: reward = 177.00, steps = 177\n",
      "00:33:11 [DEBUG] verify episode 670: reward = 91.00, steps = 91\n",
      "00:33:20 [DEBUG] verify episode 671: reward = 87.00, steps = 87\n",
      "00:33:41 [DEBUG] verify episode 672: reward = 200.00, steps = 200\n",
      "00:33:54 [DEBUG] verify episode 673: reward = 125.00, steps = 125\n",
      "00:34:05 [DEBUG] verify episode 674: reward = 113.00, steps = 113\n",
      "00:34:13 [DEBUG] verify episode 675: reward = 85.00, steps = 85\n",
      "00:34:23 [DEBUG] verify episode 676: reward = 89.00, steps = 89\n",
      "00:34:33 [DEBUG] verify episode 677: reward = 102.00, steps = 102\n",
      "00:34:49 [DEBUG] verify episode 678: reward = 154.00, steps = 154\n",
      "00:35:05 [DEBUG] verify episode 679: reward = 160.00, steps = 160\n",
      "00:35:22 [DEBUG] verify episode 680: reward = 177.00, steps = 177\n",
      "00:35:35 [DEBUG] verify episode 681: reward = 125.00, steps = 125\n",
      "00:35:47 [DEBUG] verify episode 682: reward = 111.00, steps = 111\n",
      "00:35:57 [DEBUG] verify episode 683: reward = 98.00, steps = 98\n",
      "00:36:10 [DEBUG] verify episode 684: reward = 127.00, steps = 127\n",
      "00:36:20 [DEBUG] verify episode 685: reward = 105.00, steps = 105\n",
      "00:36:32 [DEBUG] verify episode 686: reward = 116.00, steps = 116\n",
      "00:36:50 [DEBUG] verify episode 687: reward = 181.00, steps = 181\n",
      "00:37:06 [DEBUG] verify episode 688: reward = 154.00, steps = 154\n",
      "00:37:20 [DEBUG] verify episode 689: reward = 142.00, steps = 142\n",
      "00:37:31 [DEBUG] verify episode 690: reward = 104.00, steps = 104\n",
      "00:37:46 [DEBUG] verify episode 691: reward = 151.00, steps = 151\n",
      "00:37:58 [DEBUG] verify episode 692: reward = 121.00, steps = 121\n",
      "00:38:06 [DEBUG] verify episode 693: reward = 79.00, steps = 79\n",
      "00:38:20 [DEBUG] verify episode 694: reward = 145.00, steps = 145\n",
      "00:38:34 [DEBUG] verify episode 695: reward = 140.00, steps = 140\n",
      "00:38:46 [DEBUG] verify episode 696: reward = 120.00, steps = 120\n",
      "00:38:57 [DEBUG] verify episode 697: reward = 106.00, steps = 106\n",
      "00:39:11 [DEBUG] verify episode 698: reward = 144.00, steps = 144\n",
      "00:39:26 [DEBUG] verify episode 699: reward = 155.00, steps = 155\n",
      "00:39:37 [DEBUG] verify episode 700: reward = 101.00, steps = 101\n",
      "00:39:48 [DEBUG] verify episode 701: reward = 107.00, steps = 107\n",
      "00:39:58 [DEBUG] verify episode 702: reward = 101.00, steps = 101\n",
      "00:40:13 [DEBUG] verify episode 703: reward = 146.00, steps = 146\n",
      "00:40:28 [DEBUG] verify episode 704: reward = 156.00, steps = 156\n",
      "00:40:40 [DEBUG] verify episode 705: reward = 117.00, steps = 117\n",
      "00:40:50 [DEBUG] verify episode 706: reward = 102.00, steps = 102\n",
      "00:41:04 [DEBUG] verify episode 707: reward = 140.00, steps = 140\n",
      "00:41:15 [DEBUG] verify episode 708: reward = 106.00, steps = 106\n",
      "00:41:26 [DEBUG] verify episode 709: reward = 106.00, steps = 106\n",
      "00:41:37 [DEBUG] verify episode 710: reward = 106.00, steps = 106\n",
      "00:41:49 [DEBUG] verify episode 711: reward = 119.00, steps = 119\n",
      "00:42:00 [DEBUG] verify episode 712: reward = 113.00, steps = 113\n",
      "00:42:20 [DEBUG] verify episode 713: reward = 200.00, steps = 200\n",
      "00:42:30 [DEBUG] verify episode 714: reward = 104.00, steps = 104\n",
      "00:42:43 [DEBUG] verify episode 715: reward = 128.00, steps = 128\n",
      "00:42:58 [DEBUG] verify episode 716: reward = 144.00, steps = 144\n",
      "00:43:14 [DEBUG] verify episode 717: reward = 161.00, steps = 161\n",
      "00:43:34 [DEBUG] verify episode 718: reward = 200.00, steps = 200\n",
      "00:43:52 [DEBUG] verify episode 719: reward = 185.00, steps = 185\n",
      "00:44:07 [DEBUG] verify episode 720: reward = 157.00, steps = 157\n",
      "00:44:23 [DEBUG] verify episode 721: reward = 149.00, steps = 149\n",
      "00:44:40 [DEBUG] verify episode 722: reward = 173.00, steps = 173\n",
      "00:45:01 [DEBUG] verify episode 723: reward = 200.00, steps = 200\n",
      "00:45:15 [DEBUG] verify episode 724: reward = 135.00, steps = 135\n",
      "00:45:34 [DEBUG] verify episode 725: reward = 200.00, steps = 200\n",
      "00:45:53 [DEBUG] verify episode 726: reward = 195.00, steps = 195\n",
      "00:46:13 [DEBUG] verify episode 727: reward = 200.00, steps = 200\n",
      "00:46:33 [DEBUG] verify episode 728: reward = 200.00, steps = 200\n",
      "00:46:53 [DEBUG] verify episode 729: reward = 200.00, steps = 200\n",
      "00:47:13 [DEBUG] verify episode 730: reward = 200.00, steps = 200\n",
      "00:47:33 [DEBUG] verify episode 731: reward = 200.00, steps = 200\n",
      "00:47:49 [DEBUG] verify episode 732: reward = 157.00, steps = 157\n",
      "00:48:09 [DEBUG] verify episode 733: reward = 200.00, steps = 200\n",
      "00:48:29 [DEBUG] verify episode 734: reward = 200.00, steps = 200\n",
      "00:48:29 [INFO] ==== test ====\n",
      "00:48:43 [DEBUG] test episode 0: reward = 155.00, steps = 155\n",
      "00:49:01 [DEBUG] test episode 1: reward = 199.00, steps = 199\n",
      "00:49:16 [DEBUG] test episode 2: reward = 170.00, steps = 170\n",
      "00:49:34 [DEBUG] test episode 3: reward = 200.00, steps = 200\n",
      "00:49:51 [DEBUG] test episode 4: reward = 185.00, steps = 185\n",
      "00:50:06 [DEBUG] test episode 5: reward = 166.00, steps = 166\n",
      "00:50:20 [DEBUG] test episode 6: reward = 157.00, steps = 157\n",
      "00:50:34 [DEBUG] test episode 7: reward = 158.00, steps = 158\n",
      "00:50:47 [DEBUG] test episode 8: reward = 139.00, steps = 139\n",
      "00:51:00 [DEBUG] test episode 9: reward = 149.00, steps = 149\n",
      "00:51:14 [DEBUG] test episode 10: reward = 157.00, steps = 157\n",
      "00:51:27 [DEBUG] test episode 11: reward = 142.00, steps = 142\n",
      "00:51:44 [DEBUG] test episode 12: reward = 193.00, steps = 193\n",
      "00:51:59 [DEBUG] test episode 13: reward = 164.00, steps = 164\n",
      "00:52:17 [DEBUG] test episode 14: reward = 200.00, steps = 200\n",
      "00:52:32 [DEBUG] test episode 15: reward = 163.00, steps = 163\n",
      "00:52:50 [DEBUG] test episode 16: reward = 200.00, steps = 200\n",
      "00:53:07 [DEBUG] test episode 17: reward = 193.00, steps = 193\n",
      "00:53:26 [DEBUG] test episode 18: reward = 200.00, steps = 200\n",
      "00:53:42 [DEBUG] test episode 19: reward = 180.00, steps = 180\n",
      "00:54:00 [DEBUG] test episode 20: reward = 200.00, steps = 200\n",
      "00:54:14 [DEBUG] test episode 21: reward = 158.00, steps = 158\n",
      "00:54:33 [DEBUG] test episode 22: reward = 200.00, steps = 200\n",
      "00:54:48 [DEBUG] test episode 23: reward = 173.00, steps = 173\n",
      "00:55:02 [DEBUG] test episode 24: reward = 157.00, steps = 157\n",
      "00:55:17 [DEBUG] test episode 25: reward = 166.00, steps = 166\n",
      "00:55:35 [DEBUG] test episode 26: reward = 200.00, steps = 200\n",
      "00:55:53 [DEBUG] test episode 27: reward = 200.00, steps = 200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:56:06 [DEBUG] test episode 28: reward = 146.00, steps = 146\n",
      "00:56:24 [DEBUG] test episode 29: reward = 200.00, steps = 200\n",
      "00:56:42 [DEBUG] test episode 30: reward = 200.00, steps = 200\n",
      "00:56:56 [DEBUG] test episode 31: reward = 156.00, steps = 156\n",
      "00:57:14 [DEBUG] test episode 32: reward = 200.00, steps = 200\n",
      "00:57:32 [DEBUG] test episode 33: reward = 200.00, steps = 200\n",
      "00:57:50 [DEBUG] test episode 34: reward = 200.00, steps = 200\n",
      "00:58:06 [DEBUG] test episode 35: reward = 172.00, steps = 172\n",
      "00:58:22 [DEBUG] test episode 36: reward = 176.00, steps = 176\n",
      "00:58:40 [DEBUG] test episode 37: reward = 200.00, steps = 200\n",
      "00:58:52 [DEBUG] test episode 38: reward = 135.00, steps = 135\n",
      "00:59:10 [DEBUG] test episode 39: reward = 200.00, steps = 200\n",
      "00:59:25 [DEBUG] test episode 40: reward = 163.00, steps = 163\n",
      "00:59:39 [DEBUG] test episode 41: reward = 158.00, steps = 158\n",
      "00:59:51 [DEBUG] test episode 42: reward = 137.00, steps = 137\n",
      "01:00:07 [DEBUG] test episode 43: reward = 168.00, steps = 168\n",
      "01:00:24 [DEBUG] test episode 44: reward = 192.00, steps = 192\n",
      "01:00:42 [DEBUG] test episode 45: reward = 199.00, steps = 199\n",
      "01:00:58 [DEBUG] test episode 46: reward = 185.00, steps = 185\n",
      "01:01:15 [DEBUG] test episode 47: reward = 187.00, steps = 187\n",
      "01:01:30 [DEBUG] test episode 48: reward = 163.00, steps = 163\n",
      "01:01:43 [DEBUG] test episode 49: reward = 135.00, steps = 135\n",
      "01:01:59 [DEBUG] test episode 50: reward = 179.00, steps = 179\n",
      "01:02:17 [DEBUG] test episode 51: reward = 200.00, steps = 200\n",
      "01:02:35 [DEBUG] test episode 52: reward = 200.00, steps = 200\n",
      "01:02:49 [DEBUG] test episode 53: reward = 149.00, steps = 149\n",
      "01:03:01 [DEBUG] test episode 54: reward = 134.00, steps = 134\n",
      "01:03:13 [DEBUG] test episode 55: reward = 138.00, steps = 138\n",
      "01:03:32 [DEBUG] test episode 56: reward = 200.00, steps = 200\n",
      "01:03:49 [DEBUG] test episode 57: reward = 200.00, steps = 200\n",
      "01:04:05 [DEBUG] test episode 58: reward = 174.00, steps = 174\n",
      "01:04:19 [DEBUG] test episode 59: reward = 152.00, steps = 152\n",
      "01:04:32 [DEBUG] test episode 60: reward = 154.00, steps = 154\n",
      "01:04:48 [DEBUG] test episode 61: reward = 165.00, steps = 165\n",
      "01:05:04 [DEBUG] test episode 62: reward = 181.00, steps = 181\n",
      "01:05:22 [DEBUG] test episode 63: reward = 200.00, steps = 200\n",
      "01:05:40 [DEBUG] test episode 64: reward = 200.00, steps = 200\n",
      "01:05:57 [DEBUG] test episode 65: reward = 200.00, steps = 200\n",
      "01:06:15 [DEBUG] test episode 66: reward = 200.00, steps = 200\n",
      "01:06:31 [DEBUG] test episode 67: reward = 169.00, steps = 169\n",
      "01:06:48 [DEBUG] test episode 68: reward = 192.00, steps = 192\n",
      "01:07:05 [DEBUG] test episode 69: reward = 187.00, steps = 187\n",
      "01:07:21 [DEBUG] test episode 70: reward = 184.00, steps = 184\n",
      "01:07:39 [DEBUG] test episode 71: reward = 200.00, steps = 200\n",
      "01:07:52 [DEBUG] test episode 72: reward = 143.00, steps = 143\n",
      "01:08:10 [DEBUG] test episode 73: reward = 193.00, steps = 193\n",
      "01:08:24 [DEBUG] test episode 74: reward = 161.00, steps = 161\n",
      "01:08:42 [DEBUG] test episode 75: reward = 200.00, steps = 200\n",
      "01:09:01 [DEBUG] test episode 76: reward = 200.00, steps = 200\n",
      "01:09:21 [DEBUG] test episode 77: reward = 200.00, steps = 200\n",
      "01:09:35 [DEBUG] test episode 78: reward = 148.00, steps = 148\n",
      "01:09:48 [DEBUG] test episode 79: reward = 146.00, steps = 146\n",
      "01:10:06 [DEBUG] test episode 80: reward = 200.00, steps = 200\n",
      "01:10:23 [DEBUG] test episode 81: reward = 190.00, steps = 190\n",
      "01:10:37 [DEBUG] test episode 82: reward = 153.00, steps = 153\n",
      "01:10:52 [DEBUG] test episode 83: reward = 164.00, steps = 164\n",
      "01:11:10 [DEBUG] test episode 84: reward = 200.00, steps = 200\n",
      "01:11:28 [DEBUG] test episode 85: reward = 199.00, steps = 199\n",
      "01:11:46 [DEBUG] test episode 86: reward = 200.00, steps = 200\n",
      "01:12:03 [DEBUG] test episode 87: reward = 187.00, steps = 187\n",
      "01:12:21 [DEBUG] test episode 88: reward = 200.00, steps = 200\n",
      "01:12:37 [DEBUG] test episode 89: reward = 181.00, steps = 181\n",
      "01:12:53 [DEBUG] test episode 90: reward = 174.00, steps = 174\n",
      "01:13:07 [DEBUG] test episode 91: reward = 148.00, steps = 148\n",
      "01:13:24 [DEBUG] test episode 92: reward = 200.00, steps = 200\n",
      "01:13:42 [DEBUG] test episode 93: reward = 200.00, steps = 200\n",
      "01:14:00 [DEBUG] test episode 94: reward = 200.00, steps = 200\n",
      "01:14:18 [DEBUG] test episode 95: reward = 195.00, steps = 195\n",
      "01:14:35 [DEBUG] test episode 96: reward = 200.00, steps = 200\n",
      "01:14:54 [DEBUG] test episode 97: reward = 200.00, steps = 200\n",
      "01:15:12 [DEBUG] test episode 98: reward = 200.00, steps = 200\n",
      "01:15:23 [DEBUG] test episode 99: reward = 120.00, steps = 120\n",
      "01:15:23 [INFO] average episode reward = 178.56 ± 22.48\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd7zcxLXHf2d3773uvWDcrm1MscEYMGAwEEJ1IKGk8ICEQBohgfSXAA+SENJIheQlwCOhJSEktAQSCGB6M8WmGIxt3O2Le6+37c77QxrtaDQjjbTau8Xz/XzsuyuNZo600pmjM2fOEGMMFovFYqkvMpUWwGKxWCzpY5W7xWKx1CFWuVssFksdYpW7xWKx1CFWuVssFksdkqu0AAAwaNAg1tzcXGkxLBaLpaaYPXv2BsbYYNW+qlDuzc3NmDVrVqXFsFgslpqCiJbr9lm3jMVisdQhVrlbLBZLHWKVu8VisdQhVrlbLBZLHWKVu8VisdQhkcqdiEYS0dNENI+I5hLR19ztA4hoBhEtdP/2F465kogWEdECIjq1nCdgsVgsliAmlnsngG8xxg4AMBXApUQ0AcAVAJ5kjI0H8KT7He6+cwFMBDAdwI1ElC2H8BaLxWJRExnnzhhbDWC1+3k7Ec0DMBzAmQCOd4vdCeAZAJe72//GGGsDsJSIFgE4AsDMtIW3WEx4esE6jB/SCyP696i0KKnw7qptaO3M49BR/aMLV5gn3l2Lg0b0xdA+3UqqZ07LFgDApBH9Isuu3LQLi9fvwPH7DYksu6u9E4/NXYOzDxnh296yeRcWrTOrQwevY8LeffDGii2YOnYg/vLycrR1FgDGUGBAR76Ag0b0xYcn7Z24HR2xJjERUTOAQwC8AmCoq/jBGFtNRPwqDAfwsnBYi7tNrutiABcDwKhRo+LKbbEY85nbX0Pvphze/kF9eAhP++3zAIBl151eYUmi+fyfZmHkgO54/jsnlFTPGb97EYDZOZ98/bNo7SgYlf3+g3Nx7+wWjOzfA1OaB3jbT7n+Oexqz5d0jXkdBwzrg3mrt+GosQMxc8nGQLmJe/cpi3I3HlAlol4A7gfwdcbYtrCiim2BFUEYY7cwxqYwxqYMHqycPWuxpMb2ts5Ki7DHwRcCWrlpd5e229pRMC67ZlsrAGBne963fZf0PQm8jpbNuwAASzfs9PYdIXQkPzhjYsltqTBS7kTUAEex38UYe8DdvJaIhrn7hwFY525vATBSOHwEgFXpiGuxWCpFe2cBS9bvMC5vF3lzyGUce7c9X+x0MoLmzWRU9nDpmETLEIBbAcxjjP1a2PUQgAvdzxcCeFDYfi4RNRHRGADjAbyansgWi6USXPWPt3HCr57Fpp3tFZOhtaN0i7qrybqaXJQ9Q6T8nCYmlvs0ABcAOIGI3nT/nQbgOgAnE9FCACe738EYmwvgHgDvAngUwKWMsdr7RSwWiw/uL97RaubiKofhft4fXo4uVGVwy12v3MvUblQBxtgLUPvRAeBEzTE/BvDjEuSyWCw1DiuDX+aNFVtSr7PcZF3tXRAux8Ej++KFRRsAVNZyt1gsFg9TXWRd7g65bPCCffn4fbzPVrlbLJaqwNQgr6UB1TJ5RgAULXcRUeFnyqSFrXK3WCxGlGJg7mjrRPMVD+OxuWvSE6hGyCmUe7ZKBlQtFoslNkxwzCxzY7x/88TCSolTMbIK07xaomUsFoslNrXkltGRxqBwg8LnLurzckXLWOVusVi6jFrT92l0UCqfO1nL3WKx1ANl0l+hpGF1p9EZqXzuIirlnwZWuVssllgwQ5VXabdMGu2n0UFEWebl6viscrdYLEZQzIBB006gXJi0HqW7C2Vyy4hYt4zFYqkKbnthKXa1R6cgUCnOcsxa1bdv3pZOv6bRQUUpd+uWsVgsVcGdM5dj5uJgXnIZUS3GtfrTIA2rO42+iKxbxmJJn660FPckOiXN+fqKzVV3rdOwulNR7hH7rVvGYklAlembukG8rk/NX4uP3vgS/vLKCqlMhX3uaVjuXTBuYJW7xWKpIopKj6+0tHDtdk2JyoRCpoHcQTDGsGFHW6ptZK1yt1jiYw338iAqPT4emJdcNZV+a0qj/YJUyd2vrsSUHz2BeavDVhqNB9nEYRZLfCrtGqhXxKua8fKVV9e1TkMeuYYXFzs52BetM19uMEoO65axWCwVRdRBor7ibgXZcle9NnWl/jeKc48oJcvLFXGcjiNwXSSsW8ZiSUB12ZL1g6gUueUurP8cKFPt6Qd0oZpyHdwFlaZyr1goJBHdRkTriOgdYdvfhfVUlxHRm+72ZiLaLey7uTxiW1Ss3rob4696BO+uSs8fWOtUmaegblBZ7rLC68prf87NM/HLxxb4209Y18V/mlWsgwHLN+7E+KsewaJ1O4qWe0F3dJAo5V5Jt8wdAKaLGxhj/8UYm8wYmwzgfgAPCLsX832MsUvSE9USxRPz1qEjz3DXK8srLYqlzhHVVVbjc1epNJUb5Jt/fxO3v7i0JHleXbYJv3t6kW/bpXe9jr+8HP9ZePzdtd5nBuChN1ehI8/wjzdaErll5DkBMhWbocoYew7AJtU+cqZenQPg7pTlsiTBmqkBKp3fpF75wUNz8fT8dQCKboUwCzVshuoDb7yPH/zr3VTlA4DnF27A1f98J7pgCAXGPOWczWQ8t0ycRy16QDWpdOGU6nM/FsBaxpi4vMoYInqDiJ4lomN1BxLRxUQ0i4hmrV+/vkQxLEDRUqrVmOJyYPu78rBxZzs+c8drAEIs9zq4+IwVzyuXoWSWez7K516dA6rnwW+1rwYwijF2CIBvAvgrEfVRHcgYu4UxNoUxNmXw4MElimERqUQeD0v9o7ursho/NPN9rk1Fz8DQkeeWO3mLWedjKPdKhYgmVu5ElAPwUQB/59sYY22MsY3u59kAFgPYt1QhLWbUgaFkqUG45SkrPPEr/yzfo7OWKT2+VQNjQN7ttRqy5J1rnKRkUT73clGK5X4SgPmMsRa+gYgGE1HW/TwWwHgAS0oT0WKJR0e+gLtfXYF8gdkOrwvgKw0V5BmqgrWu+x0+fvPMssmVBowVlXOGSPC5x7Dcq1W5E9HdAGYC2I+IWojoc+6ucxEcSD0OwBwiegvAfQAuYYxVd9dsqTv++PxSXPnA27h31srY7oAFa7bjPSlHiiUc7nMPc1VUq1smSkczMG+gOJchwQWVXrRMuchFFWCMnafZfpFi2/1wQiMtFWRPG1BdvH4HejbmsFffbgCAzbvaAQBbd3fEruvUG54DACy77vT0BKxztNEyKrdM14gUG90zUxAs92w2k8gtExXnXi4ilbuldqiH6IQknPirZwGoFfIeeknKgiqqY83W1lhx7rUGY8yz0nMZ8jqBNGeolgubfqCO8EIhKypFdVEPCqZaUBkPU3/6pNeBBqJllAOqtfWLiD73rOCWiXMacSJr0sQq9zqkXHGzluqiWhQllyJMibV25rtGmDIg+tyTZMC0lrulZKrkWa8qqkUBloOu1hk6o4Ff47BomU+4UTHV+mvobhNxhmqGRLeMed1WuVssZaBalUkaVEvHpbPcyyFez8Zs+pVCH80jxrkzsIQzVGNkGUsRq9wtlhqlQgahlqDlHo+wJF/D3EioQ0f3jyuWETpdzVBMH1AoCCl/ayAU0ip3S11TJcZtWaia2HFXjKDlrpAvROQw5V6utLhR90eBFePcC0y03M3bqLn0A5bqo0oe9eqiiy/KrS8sxbm3dM2sy2rpuHgnU6r3gc9TCG1LOOdv3/sWvv9gaVkfvXpD2uOWN0OylZis5W4pGW4p2WCZyvHDf7+Ll5ckn5T9+orNOPeWmWjvjNaUXa3cdbeVLswxrnyDezVp9/G6xbeVe2e34M6Z6axdoB+/KFruTLDc44x3VKoTtsq9DrFZIYtUjevCkO/cNwcvL9mEZRt3RpaVrcdCgeHKB97u8vQJXIxSo0L27tfduK200VXrzFAteJ8zCaJlKoVV7pa6plpcF6Z4b18mZaXvSzbsxN2vrsAlf56dulwmcphEy4T9HGGuDr4n7d/Te8vVDagyYHcHV+6suF5sDdxYVrlb6prqfwT9eLOMDXxrsjLs9FLTdu1j7blMSrzYYZZ/MTdNeX5RbSgkGHa0OjmKRPEqNUgaB6vcLZZqwtUZJuMmsn7p6HRnUma71i3nWe4hk5hMMMkqWTa3jG4SUwHY3trplinmmakB3W6Vez1iB1SLVMtEHxMefWd1rEyW8rl1uJZ7rostd05AuavcMiG/R1jseLmzSurj3Bl2tDnKvVAodlcqWTvyBdw/u8U4Bv6io5tx7PhBScQ1wmaFrCNqSI91GZW6JIVC0T9rwqotu3HJX173vpvEdcs6hE+2aSyX5a6p1kscVmJWSKNQypR/0Mh87gzY1e7kxSmw4jmK176tM4+Zizfi7Zat+NWM95DLEs6cPDyy7WvOmJhYbhOscq8juF1hDfdwZi3bhMNG9089wdpyIcIlzxgyMX6JNin00WhAVbbcXe2Yy5TJctcqQh7nbjCJKQSjAdWy+dzViDHqBcaUHdlPHp6HO2cux4RhznLRvDOoNNYtU4dYt0wRWV88+s4afPzmmbj71ZWpt/WBXzzjfY4bFigrQpPfUG6CK/eGXPGxbu3IY+WmXbFkiYvOco9LqHJn/r9po+uIdrj+dt42L9chvGbMX+OEnm7a6SwS01ght5hMdUhhSQXrlgkiW3orNjnW9ZL1O8rabldkApTPrcN1yzQI7qAv/GkWjv350+k0qHPLuH9DFmIK3cYJv2ZM+F/ak8KNr7fci0q8wIpXXJxk1i75kxpz1aFWTdZQvY2I1hHRO8K2a4jofSJ60/13mrDvSiJaREQLiOjUcglu0WPzuVeeUqecmxwu6zSefVCMlnl+4YaS5DBBN4kprs41sdxVZdLoR3VNi+fEhPZ3tneizc1R3+bGwfPHTqXcmyqg8E1avAPAdMX26xljk91/jwAAEU2As3D2RPeYG4moPDk6LRYTZGsyRqhhKZS64r2JNRoIhXTbVMW5lzNqiNuzwXOOGQppcM3iRuCYo65Djm3n3x95ew32u/pRAEXLnSt+lXLv3a3rhzcjlTtj7DkApskyzgTwN8ZYG2NsKYBFAI4oQT6LpSR06qbcbzddYbnLVmxHp34SUzlddp7lnqBDEgmLlmHC3z8+vwTNVzwc2FcKRpY7U5fjLppWbsEr6undraFUEWNTyrvCZUQ0x3Xb8CTLwwGII1Ut7rYARHQxEc0iolnr168vQQwLZ093uX/i5pfwX//XNRkZZWTrsdTBRSPLXfrO/cM5RQimqTRcccYZM+DnWl63THFE9Y6XlhkfZ4quBvGcCgWm/F2Kyj3PRQwwakCPkmWMS1LlfhOAcQAmA1gN4FfudlWnpbxujLFbGGNTGGNTBg8enFAMi4jnciixns/f+Rr++963Spanq3lt2Wa8stT/ktlVg8zyoFpcyz2JQ0N2g/ABVdUkJlPXxc8enQ/AP5DI0d1XxdjvUuPcTUIhgy61Un7jqNBK8W2kwNTnxH97Hs6qOo9PHzUa3/3whOSCJiCRcmeMrWWM5RljBQB/QNH10gJgpFB0BIBVpYloiU2J2v2Jeetw3+yWdGSpMPLDG6cDfPa99Zh+w3O+sDcd/JWcU6rPPYk1yuVUTWIyrS1JyKG3UpFmfMOUh95ahV8/viA1ueKgTz8gKnem/F3l9Myq365bQxYfOzR6YlOaJFLuRDRM+Ho2AB5J8xCAc4moiYjGABgP4NXSRLSYUmvpbbsCrTIw0O5X3D8H89dsx/rtbZFleeQEp1Sfu4kSCyQO46GQJfjci2GN5vJzS9Ukt0zUPfrbpxZFyMcCKa3Tccuo6+j0+dzVpeTfXiVPQzbT5am4TUIh7wYwE8B+RNRCRJ8D8HMiepuI5gD4IIBvAABjbC6AewC8C+BRAJcyxqpjutYehM3n3vW0SZZ7qXHuJgpLLtKe1+eWiasA48gfpyNbuWm3l6slDmLmyVLdMttaO/Cd+97Czjb/BCUVfss9eB2veWiu5w4Ty8k0ZAnUxdGQJtEy5zHGhjHGGhhjIxhjtzLGLmCMHcQYm8QYO4Mxtloo/2PG2DjG2H6Msf+UV/zaYnd7Hj95ZJ438FLrtGzehRueeK+qk3MFfdnmssY5LTl9QKnKvRTLPUZKG0W7QRfLn19ejjdXbjGWo1iXuvyfZi6LL1dInXGv9M3PLMY9s1pwpyCHdkDV53NngfblwV1A/ds3ZDNlWwdWh80t04X84fkluOW5JejbvQGXfnCfSotTMl/882zMXbUNH560N/YZ0qvS4ijRLf1m8nbj5eoxeCblDrtrlLv8nSt3hc89rltGqPy7/3S8ruMG91Qe05mPp9wTwYp/5LOLa1yoQjZ1dXRKlrtJW6rOrjEXJ9NQOlTHPNk9BD6DUPcwlEraBnRbZx53vbJcOzhYVGjVa7nrSNuIki33Ge+uxbIN0UvlceTfzsyNIndcrusiga9blkPVvm5ugNZyT/G+KFruwTpj96MxOviCgc89KE91WO5Wudchad1Dv31yIa76xzv41xx1wFMtqPQ0OjwTJSBH1Fz/xHs4/pfPuDIw/HvOqtCom8AbhoFcQcud16Wq36BCgTjWre4tpVzeukAnY9hOW2cej7y9WpjIJlShqUOXfiAM1c/ckKUuT+hnlXsdktY9tGmns3BEkgGwNGjvLGDm4o1lbWPr7g68sWJzaBkTey3soX9s7lpc9tc38LuQSBD56CQDqqpc42L9r6/YjG2tZouBKMLctZQaGWRC2FJ+b6wM//04v3h0Ab581+t4wc23Iz4nxZWe/A0EJzFFtzNr2abAM9OYzVjlbklONQ9sJuEnj8zDeX94GXNXbY11XNh1kBeg/uwdr+HsG1/yXGbqY0waVW9+f8tubN7lpIJdu61Ve3hgAlCCAVXPclcIs6utEx+98SV8WVgQJE7dgN4t0zUZMB1Ucl10+2tGdby/ZTcAeKtdEak6SP93eRKTyan+7bWV+PJd/utclaGQltqj65NClqfBheucPNmbd5ovPQfIscnhZXkUSFgxIytas33adU8ZySLvS5I4rDhFP1iWK7QFa7dH1gukEwqpO4UkSk48tSR32+qtu71xkWIHX6ypOEkqxHJnDKY+oHfe9xskDblMSVFMSbDRMhWg/iYblfd84l4vccZg2OSU7a0doa/7cWZFhnUAJs90kqn7QcudD6gG2e66CXo1mT3yceLidQPuUb+brgNjjGnfEhhjibT7UT99KrBN5XMPuMekAVVTd5V8bo7P3YZC1i9l/nHrzCuT+DVWHLgMWrfO31uec8JSuTUVpsyS+L99+yKPVrgHErg6+CGqY/mEnZ5NZhm44zSf1Oeuu2adBYYGKYWC5xNP1E50x6lz+wRzyyQ714aM9blbapLq6rTkXB++umK0UXzgo9sstV+N8v2qiGW5u8vF9Ww0s+fiuGXyBbEzNXeJ6TpN1e/nFU1guOty3oihibo3uEBumYQ/dCbT9Za7Ve51BL/v0hu4qeyrAH8W4kohZmiMOjZsoM4rk3DiShwCCc6E7535ApqveBi/f9ofbaNTWipRtrk+956GbhnVOevuKnEsesyVj+DhOc6E9TN//2JoGzpFKc8ZAIQ499Aade2oj/K5ZfjfiAHVuJPBVOgmg6VN3Sn3Ret2oPmKhwMDGnsSe/oqez6fe8TTGDZphyMqobbOPJqveBj3vCYtsB3SjJnPXS0XALS65yMrd9m6LlruQWF4CKSpcjdZeENul/O311aElqcIV5iciAuAd33D/PGm8inxfO7+soHEYSV24nd/YSru+eJRJdVhSt0p9xnvrgUA7cSbeiZ9n3vEQ9RFhn3cByrs1VlXVVgLYvs8cueXQmraj9/0Em56dnEcEQOEhUKSYpt4zEm/fhZ3v7qiqNwVJ1N0y5j53ONFyyTLq6P1uYfM4GaI7ixl3S8PgnppJRTbot0ypZ3bUeMGYmCvJqM6SqXulLtHFQ8ulmvgk9949WK4J/VRig+gfKl1A2JM4aYvWvWiTMFys5ZvxqtLTVeiVKNT3GF05h1LctG6HbjygbeFGarBY9tDluAzkScMWZlHKXd+CXXnqNquU74myAupeHIQRU4eEw/VLdZRrdSdck/qp+0Kyq10S/FLJiLihF5atAE//Pe7JTcT93xE3WKqDOJGyyS5xmGRFmERHaK7RSyXL7CA20AnGy9n2l/+71ML8dT8td7KTGHIyvyVpZu8N+gwdNdc1TkUQxVZbLejPEGN12USCukfLE6Qx6aC1F0oZL1Yraa83bIVz763DpedMF5I1xq8A2cv34TZyzfj4uPGpdc4C3zwcf4fXwGAxMuLJf0tk/hFTZV7EplMlFGYBSnmjBHFzDPmG1/gSlF1KnyfqfyPzV2Lx+ZGK2hAHQr5hT/NijxOpyjDOlOTn5bgv55/mrlcWRcptoUNqMbxuVfDbPG6U+57Gh/53QsAgMtOGB/qc/3YTc7C0fGUe2n+xeL++INgCcTQyKP3ZZs2ETbBKS3k2HSfchCsSrFUvlDwYvpzGSrGuSuE437xUn4H3aFx0w94b9dayz24zbdYR0QXRVJegd88uVCqSxJE2BacoVr8rMrnXs3UrXKvhp6zq9G9WpZK1MMU1Z5qUopRu56LzdydAUg+d2m3riZjt0xgsC6dqx0YGxA2FIQfVjzfzjzz+dLD4tzLmf8lad26w7oiVw3g/ylfX7EZIwd0x7urtkmyFLV7KZOYKkGkciei2wB8GMA6xtiB7rZfAPgIgHYAiwF8hjG2hYiaAcwDwEMJXmaMXVIGuUPk7crWqouwlK+lED2NPPz49s6C8UCeiMlPqWo7Scy5+hDu5tLv64iTPjEEXRIwcR+TfokCY8LSehQa556P6XOPQ9IY/3gDqqW1papNvBb3zW5RLgovWu7LN+7E4vXmOforjckTdweA6dK2GQAOZIxNAvAegCuFfYsZY5Pdf12q2EWq2XDnos1evhnrQjIFxq+X+f52FVEPXFgecxPCqlft8g2oGrahVCiqV3WpmLx+ZmLkNwyFz12UCXDeiFSWu+qs57RsDRwfW0TNsXEXnwlzHwHhA6qrt7Zid8gylY/NXRN5P3oDqiaLdQh1vdWytWLpr5NgsobqcwA2SdseZ4zxs3wZwIgyyJYI/oNVo26XraaP3fQSTrnhuVTqZqI/MLWTNzPzdM9S1k3cogtFi2zdwMxUuWVCp8BrhA3TB34r2l8+LFVwHHQLbzhtFf3NYsedLwiWu+hzV4g0f41ZNsgwdJcorhuFl9e99ERNoFqxaZd23xf/PDuyA9u400nBbPIWk9RFVA36J41QyM8CEBfCHkNEbxDRs0R0rO4gIrqYiGYR0az169enIAavN7WquoQtu+Kls9Uhru/Y1TeW7k0h5yp3bt2u2dqKTe6DFav+kix3s6thmn5Ari9pxxVoQxGRX5StuMUXLVNg6Oh0Nvh97vpzWbutFW+u3OIlEoslo87Sjvk6wMcpfFFIFNwf1W6pEBD5sMgTtGqJkpQ7EV0FoBPAXe6m1QBGMcYOAfBNAH8loj6qYxljtzDGpjDGpgwePLgUMSxwHoCwrICceAOAZmV1z56n3F3XwdSfPonDf/yEcevezMyYbafnc+f1BT/zTWmthxtmuRejoPyFOgsM7XnHReH43IsWPgDlqkv/eWcNzvr9izj/Dy/HljGXUauLKOv2wOF+FcA7A/F3ahDq7qoBVZM1TVs7nHs3W0Iy9lKOLYXEyp2ILoQz0PpJ5t5VjLE2xthG9/NsOIOt+6YhaFyq2edeDpxVYqIt9ySKLzJaRqfc3UFU0bpN+8FVWalhk5h0rZtGyzBB0e5q7xQWCTeQNewNJDCgygKfVZZ7u2u55zLkuTl4kUnXPK5t762W+LmXOgoFHN7cP7A9qoOTB9O5gSGeS06IppLfBMr2LBvo3N3tzu/bvcEsbYOHIHOllHuiUEgimg7gcgAfYIztErYPBrCJMZYnorEAxgNYkoqkllCc2Yvu55CHIc9Y6vGvUW6ZsBS8JbetajosFFJzbXTrjjr7gvUxABO+9xgmjegbKaPJwF2YnOLvqvO5i26ZdCJKgixZvxPjBvcKbI/qsLOShawaUM0JCjCYC6Zy7HI7724NGexoS1aHfP5dhUko5N0AjgcwiIhaAHwfTnRME4AZ7qAXD3k8DsC1RNQJIA/gEsZYaUk3ElLV8ahlePiYz+ce5pZJvWltvDK3xjryhUTx4EmfiSQvB+Hrrqo/A8UolFKRfzNdrH7Qci+GQnrnXcZbX/WTRPncZfdH0S1T3CZa90HLvYw+9whaXcu9KRfPchclrlrLnTF2nmLzrZqy9wO4v1ShSoFHWFSjW6acC+Q6Vl3xs8jxv3ja+xx38AvQdxbF9nSWu/PAduQZdrYnDyEzVbwcf+Iws/MN6xBULpK0Ji959Uqd7tf+9ibOnDw80L7YameBeWGmDdlMlwyoqzrcSFebZuKXz3LPUmA/p1zn8+375kSW2S1Y7kmpkG6v7cRhc1q2YP/v/gfPLFjnbauxYJnUCEtHumxjMXQsns/bMBRSs1203Le1JlHu0WGt8X3umo4qJKRSFTef9pif6TiJL/5djHPPlN8tA6gHIaPuKVm5FXPgiG6Zrh9QNaGo3ONZ7mI8fM0NqFYDjDmj2dV0M1QKcbAtzNItFBhWbtqFI378BFaGxAsXa9VTzDOuLpcVfO7bFZEbURgl3FL5ymNa+kBUDvig5Z7knjNV4MF96jo6C/4ZquWaoSySxHKXlZvKLdO7Wy6wn1PJt3A+oBpXuYtY5Z6AYgKiysqRhLT9iOL6jmHPWp4x3DtrJdZtb8P9rwenW6vQuZN0biAAuH7Ge1jiTtVuzxe8xSKSED/OPb57QmXRq86Pf46l3BN2UsV9av97vlDwIlWyGUp03nFR3QvRlrs0oKpwy3xiykicPmmYbz+nkuNn3HKPHS0jYBJyWQ5qW7krXttrZRJT6q/1xgOqxb1pjQGoWhMz8XXk/Zb76ys2G9Ub9Wag26dLkRtG2ECzyi0SZ3KLSUce+rYl7vIpd03IZFlN9+CmqGshzzT2LHfhsHhwXLIAACAASURBVIYs4Wsnjvft57y0eGMCQdOBd1yl+Nyt5Z6AqNSh1QpD+n5RMf1A1IQcvj+tjjBqcLEjX8C23UXL/aM3vhSv/sSWu94a1x0T1X4SnzsP85+7ahv++Lw6MtjY5+4LhSx4+4iEOPeu1e0JfO7O3z++sEQqR8r6PnP7a7FkLAdN1i1TGXyWO99Wpjv8Ly8vx/KNyTLD+aZYp67czdwRSaJlbn9xKd7fslvfdsTxSzfsCizubELUQsqALs69uM/0lT68Q1S7RUzhaWPnrd6GHz08L7KNMNnEz50F5inCDIV3ammRUyiquKGQHfkC/vfJhXjg9fd927kSLOeAcFJKcctY5Z4Alc/dC4UsQ3ud+QKu/uc7+NhN8SxPFaXcv88vXI933vfHVxcYU/qIZRy3jGvtGba3cN0OXHrX69r9Uefy2ycXYuG6HYatFeFuo9CHXbEriTI2zRmfRPGY+OeNOxexXmGchYQ6yqkbVfHeKzfpO34gaLn/e84q/GrGe4FyWc9yTy5fuSjJLWN97vEp+oy7pqfnrWxOIdlXKQ/gBbe+ig//7wv++iAqAX3l+QJL5JYxTayVhJWbduHlJXq/arhuD/O5K9JxaeoKj3M3k0WHvAxd1AIjMn7LXXDL5JnnEiMSBlQZDCKhkpFNtOhK8ZheTTkvX4sMj4YU3XxJoqzKQbeYk5hEMtU6iama4TeD6rmYWYZBmDQtorRfPQvCLKawqvOCwouz5Frf7g3afaWeybE/fxoAsOy6033bk7plwpYb1FnoYfncVYOWcZAt9wIDTHRkocCQEaJggGAoJHeJkCBbgTF8QJi4liZJrFDxmDA9x90Xopvnm/e8Fbu9clBSKKS13ONTfG0XtrnXceG6HYl94zrS9GWm7ldk/odb265gucdh2cadWgu9s8Dw1sot8Ss1JCwXudLlzor7AjJrzl0uJy6GXGpkk2y5q9w0qt9M9Xv6QyGLE9eI/BkryzX1I4n/WNRtYStyZRUDqovXx3fnlQPRLaMadwijUpZ7bSv3iDU20141pejTLP3JSfrwrdioft12skI6n6Mt96JCMGXlpt24Z9ZK5b7rZ7yHM3//It5286yktYAFl++W55Zo88CHuTiEl5nAvuAxcr3Fzxt3tGHLrvbQ48OQo4nC3hJEVJN9fNEyjAl1B1P+xsXkvk6i3MUB1TDlnlEMqKaVUrlURMs9F9M1lWCFyVSobeXu/vUNqPr2p9tjpmVsizHpcVi3vRXHaV63xfU1w2rOF4oaL+71WSQNivJzmNPiWO0b3LR5JgtYdOQLnsI04fmF6gVdVOcaGgqptdylyULCvh/8611MvnZG6PFhyJa7/B1Qd/bF0EafdvfI55k3+JghsY5kN6rJuSUxQn2We05fgcpyL3WJxrQQlXuDJq+9DuuWSUDRcldsRDzL1IRSXSmiOCaWe15IDAWEr9rky+ceUrdozRI5VrappR01GMmvd5tmwEzk8vvnYPK1M0Jj5MXOh08DN5EpbBKTrrkC87urdJ1vkntAnuSjXiM05A1ELOer159PqOjGMZettSMvuJ+cv2HWeTamYpPrM7Hc/co9+mTiukmSkMRy58+DdcskgmeA7JpXNxPL2BQTJfGxm17C+Kv+E1kOcK+B53PV1y0+OARg8rUzMPWnT/nrSQh/iNsM8rc/+OYqR56w9oRnQmXtAjrrXFSK0S4R1XYWOLK4PS4mPvdQt0whqMD5Z7EDEKOETNjV3on9v/sornfDEvnx4crdqGofYm2NmgoYU8e5m8wE7grlKca5m3ZwvNOxlnsCoq5Z2tc0zU5Ep2Sar3gY1zw0FwDwZoxBSiZY7mEaKC8MFALOuMQGYRWCUk6R38RxFucwsZaBkFhxlVIMGXzQtSH75/Xum/gXKC9Zn797ahGar3jYd52UfvgC36eWS5zEJLqVTCXcuMNxi93vTibiMoRZwkkUlehzb8yZDKgWt3UY3EtdoTzFAdUGQ8udh09ayz0B/KbRPW9p+9xTjUAIqeuOl5YFtl1w6ys49xb9upeiCyJMTDFaRvVMhB0bdfoZz3KPXnqOK6V9ry6+mcjXVxRPb7nr616xaRc+dtPMyPKAyueu7wTiIr+d3PbiUgB+V9MVD7ytPU43FuCkH3A+PzFvrRdVZCojXyKwe6OjhEzcMkkUFflCIcPqhk8OAOgweOi6Ygao6JYZ3q+7b98fPz0lUP4rJ+yDb53irDBqLfcE8EvmW0Vd3J/yNb3g1lcApDOwGrejeH7hBm3ECOAoI5NQyHxE4rBSxhXiuGVUhL2C5zX7wnzuLyzaoNind8uYWO5JOnjdW0dbPrwT5Mf5Lffil0feXoN/vbVKIaOZkPJCFLydtC130RMTJptqQNVkPKgrdKeo3K85Y6Jv30kThgbKf+6YMZiwt7ME48BejeUVTkNtK3dF+oFykuaSaulPYooRChlmuYccGyXyTc8sxtX/fDuxcpeVoGjxxfG5q5Qix3RAVUcSt4xO9vbOAh54vQVf+NOs0LbC0imE5fyJgs8U5e4Dft3CfMrcyo+DaK2HXT5uHPx6xnv45WMLAJh1pl1juTvXpF+PBhw4vC++efK+kccc3twfPzhjIn7y0YPKLZ6SSOVORLcR0ToiekfYNoCIZhDRQvdvf2HflUS0iIgWENGp5RIcCKb8zRcYrv7nO8L+6iIqcVgpPn3n2Gifa6EQPuBaykStp+avw19eXmHkllERUO7C51nLNuObf38zmOs7ZCAybHWl4Hb/uQuX00ciy10T8dHWWcA373kLM95dqz7OQLmreH5h8I1FBV/6kFul/NqEWe4XHDXaqG4RsZMOu7/Ecr9TJJo7+5DhyuO6xufudoDub/mpqeHXIV9gICJceHQz+nTTz+4uJyaW+x0ApkvbrgDwJGNsPIAn3e8gogkAzgUw0T3mRiJKPm83Ajnl7yrJiqnm3O5h7oQk+C33YkWyMvRb7sEL5B/glPYZKv44A6o+2UIuwFPz1+GBN94PLPqhOiJsZqne3SKFQmrONknnp4sIigoZVb2JxckjH8XONq7c/W4ZnSV83hEjYy8UDfhj40t5YR0zqCf236t3sP4usNx7uG8s/C1M16EM7dOEyz64Dwb0rIwrRiRSuTPGngOwSdp8JoA73c93AjhL2P43xlgbY2wpgEUAjkhJVr2M2j3hPzpjDDc9sxhLN+xEocDw68cX+CJHyknYdPNkFKNgxFrkOnXjE14tBm4Zxhj+98mFWpdAcp+77JYJliHpjlVZ4qo1Ojm6axzoyDTXIdGAqs4tE+FP5h2zP4d7eu68PzzvDOx6Vqlbty6OO04uIhHRLRNHfPn3KzCmHJD96onj0bupfGmyGrOZgOtKvg+/eNxY7NWnG/528VH471P3S3yt0iSpz30oY2w1ALh/h7jbhwMQ56i3uNsCENHFRDSLiGatX6+efRiFd/3ce0B+cKOu74Yd7fjZo/Px2TtewytLN+G3Ty3C5QYroqeB6iYv5cEVLXffLEb5ARHj3JXRMuH7AWDphp341Yz3tBNMkir3qEU/VIS9AcV5Owpa7ubtycjXTedzb+swHFAVLud9s82WRoxirz7dvHxAsltGZ7kndX/4LXfz31hOH1JgatkG9WzEjZ86NJFsJjTlMl4IJ39zkjuZK087AC//z4kYM6hn2eSIS9oDqkpjUFWQMXYLY2wKY2zK4MGDkzXm5W7nllq843koWke+uKLNLs1MyLQRO6KXFm3AwrXbS3plFeO0A75jASflr76hxeuik61F6eCkbpmA5a4oE+caqYqG5ZbxXzdduWgB5AdfF+kT1Qmqop/+7zn1Sk5xWbOt1fvcwwuFdL7rfO5JBy6TWrGbd0ozspk6EIFI3/EcO35QaBsfPVTtxxdpzGXQlJNcV1VgmUeRVLmvJaJhAOD+XedubwEwUig3AkAwVisl+L3Gf+/ALMOI53BXh2MZdG/IGqWXTcp7a7djgZDZUI6pPv+Pr+Dk65+LtUqSfG85lmewkwummxVDIYN85HcvKLaGty3z5Dz1AGEUYdEyHJMBVa+saqfW3WJoueub85Cl1rpljJW7QaMpwO8/XbRMUn3md8uYn8y67a2+7wzOUoUKybQdSNTb8K/PmRwpR2Mug1w2g4Ys4bsfnuC0WP26PbFyfwjAhe7nCwE8KGw/l4iaiGgMgPEAXi1NRD1yyt/gzxj+w76xwnkt7dGYjZwQVQqnXP8cTr3hOX/UgNJlUNy4ICTNrQrG1Jae/DDlC6WfY9R9/Z931iSq18QtFUwToD8m6hr765FmqBbi+exFgpZ7PPfV/5y2v3ucK0uZY32L7h/X515Gt0ycjmrlZn8G1LBrr3upSCOrJH9jWfjj0/C5Y8a47VW/djcJhbwbwEwA+xFRCxF9DsB1AE4mooUATna/gzE2F8A9AN4F8CiASxljZfNzyCl/TQfFOFe6swK7NQjKPcWc7ToYdFkAixtPveG5WHWKPmPxvGWPgJjyV+cL9uQMXM/yXhsTt4xYZOWmXb6Ft4Nlg/LqnnXxzQcIuQ8S+NznrVZ31LqQUZ5/Ja7l/tfPH2lWUEJOOKdzv8hRKfLiKjrE4/ShqMHt8vJ9uutApJc5jegiVWdXqXVR4xA5xMwYO0+z60RN+R8D+HEpQpnijad6P7ps1ZnRvTHr9fxd9QqsjpYxP16t+NyHNKQdMf2ASWpeZdtlslpkC1d1OUQlwFdw0iEfv2lnu3bQduOOdixZXxxvCAuZjEJ+8HVRRTq3TJMUvWLqykgSEtiYy3jWbVRumaTWqnhY1JmcdMAQPDFvHQb1asTqrX63TNh1SOqWMSGnSHZWA7q9tmeowrPcHcIWXAijR2PpPvfWjryXq8MEpVVZUrSM2nKX/fhi+oG4A59hvvo0CCj3EjtA+fhDfzhDe42//9BcnPn7F4vHIqgwOvMFs2gZQ/l0bhluuevGknSYKl8x8VWvppx3jxR97jrlHtz21Lc+EEsu3anwa33Tpw7D29ecgsZsJpjLXXesRjYg+u3UBFVnVw2hjlHUtHL3cqMoBhIBcxdLj8ac92MlvRf2/+6jOPq6pwLbda+hpjm9dcg3lzNI633ztqvi3PmmqGgN+f6Nq2ziolprNFAmRttxfO7BY1lAYVz619cT+dx16FYKa3InFcU5V8A8HW9OGDDt0Zj13ma4Ba+Lc1cp/T4ha+tyxMOiDJiGbAa9uzUgmyVlMIAOXYeUhuVeCy4YFbWt3AOWu2z5hR/PczR3a8gIA6rJbwZVYq+v3P1GYBtjwBm/ezGwPVa0jPTdl9s7xOcuXqOkIYvlcl3J/lFVM3Fi4UuZKMYQvH8em7vWyFwwdY/8ws2fItOUS+aWMbUmufJuymXQkM0gz5zZ3XycRx8tE6xftSrRWZP39n0XO7uJe/cBAAzp3RQuYyajeJNTlyUibYeaJGnXOVNGSLJY5d7lyD73uAOq/CbPF5jgc09Xc/17zmrjsqUoTQYxzl2sU3bLFEvc+sLSmG3wzqM82j3YOavebszrUxU1teQYU3e2Rm6ZEnUBnzDjpVEw7INNo1m426dbQxbZDKFQYD7/dpxomazCyp86dqDvu6h4f3XOwbjvkqMwrG+3UBmzGZXlri5LCF7zhizhrs8fiWP2cebQHDqqX2h7IqMH+iciWcu9AsjWtuyGiXLL5KXXUacuddkksyfjEqeNoMtEnfJXfkBKdWvErSMOcthaKW4V3fGmkXG6zJ0mHVupYXJcucYeUI1puXdvyCJLhM5CAb2E6ftxfO6qjqCnlApAFKtHYw5TmgdEvt3kMhR4k4vjliEQpu1TnMB0+JgBoe2JDJTywuQSLC1YDdSm1C7FQVDnb1zLnQ+25Bnz6gibnl4qUYohThtyLnbR5+7P/y21UWCxrF+Rv7y8ArvaO40syST6LRgt43zv36Po1z3+l89g8Xr/Qt1xMH7rYJrEYwaHlqrcM9L4j6nIpjqIr2PavTGLTIYwc/FGX+itNlpGsV2t3P3Jxfj5NAmrMEW5OjIUtNyjykdx/5eONqprqPRWYS33CiCn/JWV4zn/NxOL1uknA3FLOV8Q/dWaAVDDJ+z2F5fiV4+rfam8Ct0CBHFfDvjal/zY4mu8YLmromVM9Zui3Pw12406oSR+yj88759aXygA+w7thWYpX8fdr6yIXTfHZPFugPvck/nsS3XLiC7C215Yiu/cb5bvyLRT4ekGmnIZZDPANinTpk6ZBaJXNGV7NPotd16kd7fotwNOLkuBSJcCY7j9osMDZYmC5656az9sdP/ANhV79fErd1HuWqKmlXsxWKaopEV2tefxk0fmaw/nN09nQR1GKKKyVjfuaMPX/vYGtrcWc2D84F/v4n+fCuaiBoqd0LbWDuX+WCP7BPzmyYXFuhlTdnKqaBnTjkq3EEaSWG8Tnl7gTyDHwECgQEdRivKUI1R0vt8fPzwvVspgERPxLvvgPtp9WcEtc+2/3zWozX9cFL3d/OIZIuXgKb/e8nXe0RqM7lENsvaUlPtQV1mKnUhDRGhPNkOBAf8CYzhC4V5xlDukss7fJJMSh0rKvVIrKZVKTSv3YH6VYJndmkRgPus2X5yduGDtdry8ZGOgvEoh/nrGe3jwzVX455vq9Dn3Sxn8eBVbd6uVeykDlU5WyGInt3VXB655aG4g9j5fYAHfvs7KVonTmWdGbxhp+CkZU88+LCXGmC9QwdFZuw+9tUrZ2X7fXbw8DBML+tjxg3DwSPUgH3d/xHUFmvan3M2VyQCqqEeu8OX7Qhe6KbPPkF7eNH0AmOguNycq659/fFJoHTmlcteHaQYsd+namayn/O+vHIOLjm5G/x4N+OqJ43H8fs5gbDXkZk9CbSt3929Y/HWrZop3p+S6EJ9j1ULUqgedLxzRU7H02O72PL5171u+bdyK0GWeLCUUkoF5bxcFxvCLx+fjjpeW4QF3ZXtOQREFomtXF21SLstdpsAcRS53FH+euRzz16gSSEWzU1JQUXpYXuk+bB1b0zoBoCGnf/S4onp6/jptGVUbpm6ZgT2bvPKq34kb1fK+7YbKPZMBvvvhCRg5wFlIuldTDp8/ZgxuvbC4kPSwvv5FpmVlnCHCrOWbpTLq0EuVrPIdGmbBf2rqKPzx01Nw4PC+uOaMiSAifPPkfTF6QA8AxetVa9SmM8lFTvmrijbRWe6iso5Kg6urmyuKRsWDKme0A4AHXQtf537RDVSqZFNNMOKl8gXmRZ7IftICC1rezPXXqyZGyeRZ0PJXkU5ssBOiKj+4uzvymH7D84lq3Nnmvx+iYv1zmQw6FAtZTx7ZD3NatiQOX23MZrS2JA85vHPmcu3xWSJ0Sj+QaYc6wHUzODM79cc4nWrx+qjcMiq4lVwcE2O42s2maMpaISXx2EE9sWTDTgBMOahLIO2EOxN+dJZ6jdOLpo3By0s24Qwpbr9WqCvLXWWB6mZhimU7C9GuBpW1yi0ZVQeicr0s3eDkLtEqd50FbXCjcgUNOOF+/GYPhEIq3DJO26o6gxs78wUjhZbEcpf93wX3PNKcRLJbclMN6dOEvt0bAhNXOLq2f3PuZK0taCItX9pOebxBBSr3RJiiHju4p/cWwkP9SGO5c6Us7zJ1y8jHJfE2ikqcn6s2SksxoBosEv8eGjOoJx77xnEY1Ks2LfeaVu7FTI4Oqpto9dbdeG2ZvEqgf9HifKEQabmHdRyywnDq1Neny3cRlo5WJhAKCXFQmIGrGJVyV8mmTIegaLets2DklkmikOWOmDFnQLWcoWhNuSze+v4p+PnHD/Ym94ioJukAzvXXXYZPTh1t1K7MwJ6NGD2wR2CdWBWqCUVRseN85SyeMkD1VgQU34TlhFkXHd0cKZdTr39ANsnLjXh+3C2XJP3AnkxNK3c52ZdKQbV2FPCJm2cGlLc4QUJMpqVDZTVwBabyoYfdiDrlHtei95dRXwdVfg5VR2U6YaetM2/mc9coxTDkZecYHAWkG0RLA7FmlfGn66TCDMUP7DsYZxwc/irflMvgk0eO8m37x5en4dlvf9DzVYehUuRh+k3cxePNdT53/vOK1vDdX5iKj7jnNKxvN+VC1V5bFGwzLuI91iCtghTWpkyZ5tvVBDWt3DlhA6pyGY6o4EwGCWWFuLs9j9VuKleVcg/Lphs3zl0lmmqGquhz99wyijh3lVvG1HJv7SgYTWJKEi3TKlnuC9fuADThemkhXkfVq73udb/UWPamXBafmDISf7t4KgBgeL/uGDXQGcAb0b8HPnTgXhg/pJdX/gvHjvEdf/4R/o4BcKxdnrslKG9R4PFDHMV8xuS9lW8A/A1KNNzF9NAzrzwRj379OO25kWe5u2/WCTSseAjvYHUmmDh2IHfGPHRy2j4D5cPqnppW7qql5nQEc6zE9LlLBb7wp1lY5ebj2N0efI0Oc8uIg0Uia7aqtyvXjZS+M8FyLzDm7VelH1BZ7qptqsHGtg4zyz2JW8YZCHbanLl4I97fshtvrdzSZYmbVApbPxW/NJl45kdef5M0KJ/LZnxveHIHd/n0/TH/h9Px3o8+5G0jIjx46TRle6K0zYN6YN6103HB1NHKNwAePit20B0Jksx5Y2IGZSe44ZIc8R7zfgNNRaMG9PB+D/n3Orx5AOZdOx3Hjk+2TnMtU9vKHX7LIMyilHVt3GgZWUm+sGiD9zmuW2bDDnU43aV/fT1UBhG59rbOfHFgucC8cYDALD+Nz31XW/AcVMvltRr63JP6QLnVyAefS6lL5PbPHI6vnjg+sF20aLvWcvcrd9n11JAhX6STPByQyRC6NWR9kVrZDCkXlpDplsuie2PWDTNVKHfPci/u2ysi0VcYUbfLq1edGJicJBob8spUIqdOHIqxg4tvOKp7pbsiVDktqnn2avVKZoA3YBMSLcMJs9xNpuSHKTRVtEwaeaTD2pY3XfKX171p0y2bd2Oxu6qQ7AISffMiU3/6pNGyaW0dhgOqCf3kbZ0F9GzyK880LPfBvZrQuyl4u0f53HUeoTDLnSh6nIR3KrweecZmLku+RGqi++SFyz+okSm0yWI5oWC45e7s690thwOH9w2Ui8SrOvxaDOkd7DhEQ82LllFUwycYceOsKwdWX/2fE71Vs6qRxJY7Ee1HRG8K/7YR0deJ6Boiel/YflqaAvtkcP8Wo2XMfe6dknKPehjl/aLCMXVzJEVVlUreNa67R4w66cgHO7U48fQyrZ15I597Uj85VyziIxrngQ2zpOQJSTIqha1Lo0vQ5yQnUGCylA5uBMgWd05aiUi8niP691DWFRYto+uLVOfXx01PwK+76PuPgxyqHAfx/pajZZ757+Px36fs62xzLxG/fgfunaATSsiQPt3Q12CxkkqRWLkzxhYwxiYzxiYDOAzALgD/cHdfz/cxxh5JQ1AVxZS/zvewQcwP/OJp3/eC5HOPugHlusXXYZWVnmaKYBPLXYc8iemOl5ZhpiK9AmD2tmFsuSe0oLhyFxVtnLpOmbCXcjuRei1M/4Bq8DidwiQiPPzVY7VymIQzAsU3q0ap48llyJcqwWSVJTkEUUQX5y1f22s+MsEbfOTKPfnaqf5QZZnBIQt2+KJlsv56mgf1xPihzqDwoN6O5d63ewP+dvFU3HzBYZFy7SlRk2n53E8EsJgxtjyl+oyQQyHDlM667W2Y+pMn8e85zixRbrk3uMt5ycc+NX8t9rv6P97EDVnxiQ+FciZnisr9oGseD2wzzTuiyuSnw2S9yVbTUMiET9DW3R045+aZ+NmjxYRvpm8BZ03eG8P7qX3DBFLGsYtKjysxsZzWcie9ciIqKvfpE9WdDYe/WcnRRblMBq1CBkuT1Z2yGhcPl+nZbx/vRed47UidyokHDPXcHxkqUblH7P/XZcfgT589QrlPvBX5m4RoJZ8yYShu+K/J+NqJ+3rbpo4dGGlJ//ULR+LZb6vdWvVGWsr9XAB3C98vI6I5RHQbESnzbBLRxUQ0i4hmrV+/XlUkEtkyiFI6a7a14uePOul4ufJtymVd5e4v++sZ76Gts4Albu5wuW4xw53Sci9zgK1p/VHrpIqYdASO5R5dV1LLffOudry6bBM2CjlcTP33x+83JHRAMaoersM+dthwb1uSaBlHuTszlE+aMNSozf49/UpJdiGZzLDk/UOT5hqMHtgzsEqS3BHwlZlE2UodPNbdqnv17Ybj9lVHsYguwhMOGIJrz5yIq047wNtGRDjrkOHK1B9hHD1uEEYOULu16o2SlTsRNQI4A8C97qabAIwDMBnAagC/Uh3HGLuFMTaFMTZl8OASw5QMLHfOAcOc17micudrNfqPfed9f2KqMEtc1W4MgzkRpi8GrYb5ywGzt42klvvVpx8QuW4mAGzaGUzboOooVDpXlUFS3McV2RHNxcgM0WAmheWry0AZpe+45S4uNKLiqLED8a2T98VPzvbnN5E7IpPUtd7grKvwzpq8N/4uWeoygYHcDHm/Lz/1pJZ7U0iKhSjEe7EhS/j0Uc2BFZ4s4aRhuX8IwOuMsbUAwBhbyxjLM8YKAP4AQP3elRJEYsKs6PI8VzN3QTTmMugsRFujcWLonfrLrN0NkVP+hiEPvqpo6ygYdQKyQv78sWMxvL9+5uW9lxwFANisyLqoUu6q6fvZDGkHTR3lzqMuivL73TLOX1Hh6V5AohTeLve695OUe+9uOdzwX5OL9WQIXzlxPPr1KH1pt6Jbxvn7jZP3Ra+IUD35emUyFHgWTEX556XTcPXpRev6pk8ehi8fPw77Do0/IJumW3NPJY2u8DwILhkiGsYY46tCnw3gnRTa0EIwm6HKyWUyuPOlZZ4LojGXwc62aGs0bL/qRpTXA1WhWgQ4beIod5MOSYynD0PlJw87bsro/shlCJt2BZW7yt/cmMsEcvpkI2azNoTES/PjxXJAyNhBiG4nEO754lF4bO6aQCd0wv5DcNYhwzVHirIG3TI/OfsgqTRE9AAAGphJREFUdG+MTjimclXo3kACbhmiwPUxtdwnj+yHyUKO+pEDeuA70/c3OlZGFGHlpt2xjv3S8eNwwv5DErVbT5Sk3ImoB4CTAXxR2PxzIpoMx6BeJu1LHSIKTfkr89LiDbjtxeLSe025DLbt7ogdLSOiGohsN3iNKLdfHlAnNZP5xccn4dv3zUFHZ3rRMlFhhzJEhF7dcsoQQlVz8oxOXoc2FwyKE3zyDOjdlMP2tk6fP5krQDFyRacUeTPfmb4f2jsLuOGJhcIxzpJuh43uj3mr/e49075cNXZwvpSLRkZ2K3Xko6PAAso9Szhnykg8MW8tzjt8FL5z/5ySFkdJCr/HJo/sh9MnDYt17OUJO5R6oyS3DGNsF2NsIGNsq7DtAsbYQYyxSYyxMwQrviz4Lffo8vPX+NdUddwy+jh3cdanDtU+k8HJrkhqZGK5c/9uuyJvuczmXe3488vFoKgDhvXB9z8SzNWtTCUboSOc3OmqsM/gNpV16szQNHDLFBh+98lDA2W40e+z3HWGu3syXz5+H3z9pGLExoj+3TFKGLCTrV7TDr2UiVvXnnEgmgf2wIj+3TFucC8M79fd5y4Rka9jlgiDezfhH1+e5oUZViJ0kD9Sf/3CkTWbcrfS1HT6AcB5eDyfewJt2ZjNGM1QDVOSKhdM1CIQXYXJgCr375qUXbhuB95YsQWAM4nnP187FmdODroZVMop2oqkwIzasybvrey0Vco9Q+FKkSttXboJ7n9vyAXdMkE3iZoXLj8B3YRZi7IBbjr/oRR33THjB+GZb38Q3RqcNAMvXnECpu0zSFlWPi+xU+Zeuii3zGenjcEBw9QJy5LyvY9MQFMuoxxbsZhR+8PPVLSGkmSfa3SjZaJyqatWVuJ0KHzVJoOTlWRw7yas394GoPiAxwmbBIAHL3OSVMWZ/BNGLks+d9agXo244dxDfDHvHDEWvSFL6Mg7q/TofO4Ev8+d/zqi4uIii2tmcgtdXpFJ/nVPmTAUc1q2QkZ2aZgqbdOFMUpFHrj1KXf3mYj6Kb+neHMrlfOOGIXzFJkv02Jw7yYcNkoZpV031LxyJ8B70pJYOzzOXdcv8Bt83bY25f4jxwxQJg6rFstdB39gnZWOnAdczqcuc86UEbhnVnHR7+KMSMXTn6Bva8hkfG9IPOGTquMVk0E5cjBkKEa0DCtu99cD9GjMYtKIvpjTstUbZJXfCOQJUbd8egpUyJOgWjabDQ7yUMrGbAbt+YJRKGQS5LVcRXkL3jWqvymdr111UqVFKDs175YRQyGTvMk2uT53nUuHv0av36FW7lkpex9nsyLqo5rwrXTjKr33t4Qrnm5SkqSwSS5JBotzWcJuwTU0vJ8TOqmqqrvP9cFnUobNjCW/W0ahLIuLTBTLqrI2Lv3pacaTZ2SXxpFjB2hK+tnmToIyyTp40gHJI0PktAfiGxcztNwt1UkdWO4kpPxN5pYJO5a/DWzUpOnNZkgZLXPHS8uMZdh/r96Bgd5y4y2ogKLl/u375oQeI0eo8A5C5ZNVXZOoXyeXyaBVeAsa6SbIUv023RTKPRsSLSOWK7Bih6EqLfru+anlDCY2qeBej2yG8Ob3TkaPRrNH7pCR/fDA6+9jwt598PzCDaEzVG/+1GGxXWocVaoCjpyGwFJb1IflHiPOXYYr944I5a6LfiEKDgKacniz4/PrU4HMcqKr1XR6v2y5Z4QOQibJb9GQJV/oJs9+qHqrEnW4lwMlQ9rJP0TFzsg3iUlQXNy91rtbQ8Byb0hovorXqHe3BuOcO5+aOhovXP5BTNCsrCSSy2YSz94MU+57u3l6Jo3opy1jqV5qX7kDJUXL8NF43UozvE79xBezhFtjBvUMbLvjM0fgmf8+PvZA8GUf3CdWeRVZwV8u+6l7ahY3kC33sMRSqvGPKLWWy/onJvHEXOrLU6zNs9wzFLKgdVHOfIF54XX7Di2uBcr93H27N3jXhNcm+6ZNyUpvAKYQkTa1b5qEzUc4ZFR//Odrx+KLx40tuxyW9Kl95U7FVeiTxI1zhaWzzLni1g3WZogiZ6MO79cdvz7n4MD2nk05NA/qGXusQNVRqAizEsX1LWVr9380MdEByz3D6wqWVV3OaLcM+RY+GeAm01IuM6iy3MkZlNXBdxUKDAeP7Id7vngUvnVKMUadR6j06Z7zLFr++yeNOxf9+NVI1L13wLA+iSKfLJWnDpR7MalSnPS2HO6W0VnfhQLDx256CU8vUGeuzGQoctr+1LEDQwfgROX1m3Mna8tx5GXD+MCjTNjqOeLzKncC3XJZz2UkorPcVasm8XP6/DFj8J+v6fOeizRkM75B3b7dG3116eUvyqPr0Aqs2DkNc6/XEWMGKN0SjuXu3hduxx3mvggjW9TuVUlbp9OZTh07wMvvY6kPal65b2/txO0vLsP21o5kyj0bbbnPXr5Ze3yGomPaMxSeCErsV/bfK9rHqpp6r0LObT1x7z749qn7ARDdBRRQXLksKV0t8pJiKrcM99/zN5399uodOcHlsa8f5zuWwyNFejUFxyRES5hf2wzpZ6i2dxYwqFcTfn/+obglYkGH3t0avHp4x51UuVf7YGSbG500ekBPHN5sFsljqQ1qPlqG8+6qbUbJumSi3DJR8erZTPSAajZDoSvpiD53kxA701l7snKfNKKv9wBzpdOrKRdQiBnSKHdJNpVR2pB1Fpng1raJUtxvL8fvLXeAfJGGr7kLW9/87OKijELRjGC569wn/Pc1yVPiZJcs5mcBkq8Jy10aSVU8/63j5uqJS5/udaMKLC5184sWmDpZF58EooPnnP77ayuV+3m8sQ4i0kbaiGXCLDjRn2+k3BvUSlZmYE9/Gtm+3Rt9LoyrTz8Ax+83JOCndjqjYKWyz121DFuD9CbkU+6uwj//yFH46ysrAvWLivmw0f0xcoDjPunemMWXPjDOp9xFy90LycwAOc3LqEnGywcvneYl+mqQLfeEa8ImHVDlfOkD49CRL+BTU0cnqyCCsw8djuWbduHSFAbpLdVFzbtlOAxMablf/qHwDHHcLbNBE8f+y8cWhB6fNQiFjOOWUS0FJ9PN0HIf1te/5NyI/t296fkMTo71fYb0CnR+2QwpB9ECljv/q/C588lIKotXNz4nlv3stDG+MMWAchYHVIVORqdE2w0yXh48sh/Odae8yz533v7ogfEiWLxrlNB2796YxeXT9w90rGnRlHPq72UXwqg76ke5M7VrJSrKoTFCUW7eFW65Zyg64sBRlvr9sd0ykuX+pePHYYLCr72XQrnz6yG22SwprCxRQAF/9cTxPgVz/pGjfH57zq0XHo6PHTrCcwmp3Am6txjxt9rV7s+t0r9HI86ZMsL7zkv2asp59aneNo4aOxCnHbQXDh0dL1abK/fpB+6Fkw4Yit+cewjOOHhv3Hrh4bHq4Ve5yl3vljqkzpR7UMtGTRoxHZzUYRImFhbFARSjQQ4Y1sdIHrnMwJ5NuPWiYG4TvuoU4EyYOnLMQKWSyWUz+Ogh/nVD5Zwo3zx5X1+7Pzn7IOVMzYNG9MWvzjnYm1Wq8rnrlLuYlVLO15PJEH7+8YNx0dHN+Pap+3l1/PjsA/GTsw/CwSP7YfTAHoFw2H2G9MKNnzwsdnbBz0xrxn5De+P8I0fhjxdOwcgBPfDb8w7BPkPirSrUoyGLI8cMwO/PD6YYtljKSd28ixUYS2i5l6bcZSUoMrxfd7y/ZXfo2p5A0fL/zbmTjdwyKkWlUpjD+3XH2ME9ceWHDsDJ7kLNOsUqdlLZDPmms3/9JGdAM45r4LIT9sFbLVtizW7c3lZ8S9ItLH3NGRMBAHNatmDGu2tx1LiBGNK7Gx68dJqyvOmMUJkR/XvgsW8cl+hYkUyG8Pcv2hBDS9dTN8qdQT1oFmVZJw1x44Tl9Dh14l647cWlyBKFDsgVU6uqfd0y3RQLD6t0dlNDBk9963hlOTl2PCcp943uWqa//MTB+PhhjjskzqIJU8cOxNvXnKrcp+tgNruLY//ps0doY/c5k0b0w7wfTo+UI6lyt1hqnVKX2VsGYDuAPIBOxtgUIhoA4O8AmuEss3cOY0wfKJ4SBcaUg2ZRlnupD39YFAavOpOh0JXguZ41FUW23ImKCnNAz0ZschWz6q1CN7AndioZKma6FLMvDu7dhPFDemHkgNKmxev6U55Jk6cdSAKPsBnUqwkbdrRZ5W7ZY0nD5/5Bxthkxhh3+l4B4EnG2HgAT7rfyw5jTKloox7uUtcxDYutF6M4wiJcRMvdBFXUiiopVti5y6ctpwDmyl1+S3j8G8fhtoviDSp6bbp/defJO6UhJSj3fj0asey60731Rqt9EpHFUi7KMaB6JoA73c93AjirDG0E6Myrfe5Ryj3JxCcRXc6ZA4f38VwgGQp3D/E6TBVRJkP40VkH+rbxlAQfFibpqNrkTchSZ2XL3XU3Bd8SSleWut+ETzDq36NRuT9WG0K+GYtlT6RU5c4APE5Es4noYnfbUL4otvs3+UoCMegsMGW0TJRbxmRyS1S7Mv/+yjG475KjQ7Mmini5xWMoInlws1tDFm9+72Rc85GJ3jZVu7omDhlVHPjMZYoTs8LcSUnp10Od4vhnH5uEN793ciqJqkocSrFYap5SH4FpjLFDAXwIwKVEZBxeQEQXE9EsIpq1fr06KVccOvIFpeUepVhLXQ6Pdw48ZwvgKK9uDVlPaUcN2nJXShz/8Mj+wQHHfj0afYtKqHzuPHf8ISP9USziItdiSoVSQ0VVDOipdrs0ZDPol4LVDghZL1OpzWKpPUp6chljq9y/6wD8A8ARANYS0TAAcP+u0xx7C2NsCmNsyuDBg0sRA4DjXlG5WKJygpjkYo9qF/DnBS8u5+YoSFW45cGCcuUTkGRr/NjxwRXreTKtI8cO9KJXdK4SVYDO0D7d8PBXj8GPzj4wsI+nK3AGVJlSpjTor7Hc04TL3aNMMzstlmoncbQMEfUEkGGMbXc/nwLgWgAPAbgQwHXu3wfTEDSKzoLacs9G5ASRk2vFhfvLRQXOXUGd3kQev/K9/0tH+ybD/Oa8QzBv9TYMkHLB7DOkF55fuMH7PnXsAPz2vEO87weP6Isn5yv7TgD6GPyJe6tTAReXlCMvJUEcy10VoikyckAPzGnZir0jwhzT4OOHObNkS1lf1GKpZUoJhRwK4B+u1ZgD8FfG2KNE9BqAe4jocwBWAPhE6WJG05FnygRhYZOMAOBDB+5VUrsqBZ6LyAV+2Gh/rvReTTllulX5TeSAYX0wpHdx1unAXk5n0EOzclLSMMAMiW4ZM8v37i9Mjcy78rOPTcJHJu0dmQI4Dfp2b/Di8y2WPZHEyp0xtgRAYHkhxthGACeWIlQSOvMFpVsmSsEREaaOHYCXl2xK1C633EULV84omDRdrOwykheK/v5HJuLgkf1w9LiByuPjR7YUc7TwpqKscc5RGhlEejXlMN3tTG+9cAo+d+esmPJZLBZT6iamwImWiR8KaUJYHVwBN2aLFi7PAMk7m7hLtPFokrwUySOvEduzKYdPHjk6lfBEEVHeuDlZTDnxAHV6AYvFkg51o9w78gw72zoD202Ue9Q8pjDXTl5hnTdIqxGFpftVwafe80Ra/BwSLDSVCDEUsdTcOxaLpTLUTW6ZG59Z5C1wLBLXct9/r96Yv2Y7AGdQ7r7ZLU4deXV5lc+dW9Le4sruvps/dRjWbmuNlGFE/+6Yu2obWjbvAuC4fHa1531pessB78PEzqyc0/f/74LDsFpYM9VisaRH3Zhl21s7lRZ41IAq4J889JtznWiU8UN64dSJjn84zK3CrXNVVE5xn3P89AP3woVHN0fKc/XpE/DB/Qbjt+cdgqPGDsQlHxjnqy+KP356ii/3uSnewhIE3HbRlLIPSJ46cS9cNG1MWduwWPZU6sZy12HijhY7Be6GaOsseJZy2ICo6Ff/xccn4YHX3/f28TGAuD73kQN64PbPHAEAuPviqbh3lrMEoGlI/kkThmpT5prAGHDC/kNxwv7WL26x1Cp1Y7mnwV2fPxLD+nZD94YsvjN9P292I7fK99+rN34sTf7JC66XT0wZibsvnhrYFxVrHwW3/EtNcmaxWPYc6l65x8kKSOTMbJz3w+n48KS9ve05KakW5/PHjMGlJzgLC6uSXck+96RkpUlR5eIrJzqLcuhyv1gsltphj3bL8EFQb51LKa0WTwvw0UOH48ZnFrtlHM6ZMgJXf3gCAOACzcr0xWiZ0pQ771DKbblfMHW09lwsFkttUfeWu6jcv3z8OCy77nT8012SLSqh18gBPbDsutN9VjxXtCZ6lvvcS4048ZR7mS13i8VSP9S/ckfQpcIjaLhFHaV6vRzorFjYRM92Joxzl+F9kPW5WywWU+peuTMh6Wtx2TvnL7fco1SmaP17lrtBMtm0fO4j+js5W+IsNm2xWPZs6t7n7oNb7q6Wj6t0GRj27uck7ho3uFdEacdn/9bKLV4q3aQcOLwvZnzjOKM2LRaLBdgDlHvPxuIpcsudu2Vkn7tJYM3R4wbh3kuOwqGj+keWveaMCThnygiMHtjTXGAN44V88RaLxRJF3btlRg7o4eX05hY0aZS7zqUtR9Ec3jzAaJC0KZfFIQadgMVisaRN3VvuAPB/F0zBA6+34KOHOtPp5RBFrvSj1gu145kWi6VWqHnL/a7PH4nzjxwVWiabcWaPcmvbSwvgWu7XfXQSrj1zYmBdUfF4wEmxa7FYLLVAzSv3afsMwhGKVYzC4JY7X8Gob48GfPqoZm1e9HGDe+Ly6fvjpk8dWpqwFovF0kXUhSmaiTlJ6KDhffGFY8cYZyQkInzp+HFJRLNYLJaKkNhyJ6KRRPQ0Ec0jorlE9DV3+zVE9D4Rven+Oy09cdXEnd6fyRCuOn2CtyiGxWKx1BulWO6dAL7FGHudiHoDmE1EM9x91zPGflm6eGaUc0EJi8ViqUVKWSB7NYDV7uftRDQPwPC0BIuDyYIcFovFsieRyoAqETUDOATAK+6my4hoDhHdRkTKQG8iupiIZhHRrPXr15fUfrbE6f0Wi8VSb5Ss3ImoF4D7AXydMbYNwE0AxgGYDMey/5XqOMbYLYyxKYyxKYMHDy5JhlJT6losFku9UZJyJ6IGOIr9LsbYAwDAGFvLGMszxgoA/gDgiNLFDMe6ZSwWi8VPKdEyBOBWAPMYY78Wtg8Tip0N4J3k4plhB1QtFovFTynRMtMAXADgbSJ60932PwDOI6LJcDLpLgPwxZIkNKDUlLoWi8VSb5QSLfMC1OtcPJJcnGTwHOvdG7LY3ZHv6uYtFoul6qj59ANAcaUjnk7AYrFY9nTqQrlzn3t3q9wtFosFQJ0odx4s073BKneLxWIB6kS5t7p+dtktY900FotlT6UuskLyhaj79mjEPy+dhrXbWrF1VweOGBMvFbDFYrHUC3Wh3A8b1R9fPn4cPjNtDAb3bqq0OBaLxVJx6kK5ZzKE70zfv9JiWCwWS9VQFz53i8Visfixyt1isVjqEKvcLRaLpQ6xyt1isVjqEKvcLRaLpQ6xyt1isVjqEKvcLRaLpQ6xyt1isVjqEGKMVVoGENF6AMtLqGIQgA0piVMurIzpUQty1oKMQG3IWQsyApWRczRjTLkIdVUo91IholmMsSmVliMMK2N61IKctSAjUBty1oKMQPXJad0yFovFUodY5W6xWCx1SL0o91sqLYABVsb0qAU5a0FGoDbkrAUZgSqTsy587haLxWLxUy+Wu8VisVgErHK3WCyWOqSmlTsRTSeiBUS0iIiuqLAstxHROiJ6R9g2gIhmENFC929/Yd+VrtwLiOjULpJxJBE9TUTziGguEX2t2uQkom5E9CoRveXK+INqk1FoN0tEbxDRv6tYxmVE9DYRvUlEs6pYzn5EdB8RzXfvz6OqSU4i2s+9hvzfNiL6ejXJGIAxVpP/AGQBLAYwFkAjgLcATKigPMcBOBTAO8K2nwO4wv18BYCfuZ8nuPI2ARjjnke2C2QcBuBQ93NvAO+5slSNnAAIQC/3cwOAVwBMrSYZBVm/CeCvAP5djb+32/YyAIOkbdUo550APu9+bgTQrxrldNvPAlgDYHS1ysgYq2nlfhSAx4TvVwK4ssIyNcOv3BcAGOZ+HgZggUpWAI8BOKoC8j4I4ORqlRNADwCvAziy2mQEMALAkwBOEJR7VcnotqVS7lUlJ4A+AJbCDfCoVjmF9k4B8GI1y8gYq2m3zHAAK4XvLe62amIoY2w1ALh/h7jbKy47ETUDOASOZVxVcrrujjcBrAMwgzFWdTICuAHAdwAUhG3VJiMAMACPE9FsIrq4SuUcC2A9gNtdN9cfiahnFcrJORfA3e7napWxppU7KbbVSlxnRWUnol4A7gfwdcbYtrCiim1ll5MxlmeMTYZjHR9BRAeGFO9yGYnowwDWMcZmmx6i2NZVv/c0xtihAD4E4FIiOi6kbKXkzMFxad7EGDsEwE44Lg4dFbueRNQI4AwA90YVVWzrUv1Uy8q9BcBI4fsIAKsqJIuOtUQ0DADcv+vc7RWTnYga4Cj2uxhjD1SrnADAGNsC4BkA06tMxmkAziCiZQD+BuAEIvpLlckIAGCMrXL/rgPwDwBHVKGcLQBa3Dc0ALgPjrKvNjkBp5N8nTG21v1ejTICqG3l/hqA8UQ0xu1NzwXwUIVlknkIwIXu5wvh+Lj59nOJqImIxgAYD+DVcgtDRATgVgDzGGO/rkY5iWgwEfVzP3cHcBKA+dUkI2PsSsbYCMZYM5z77inG2KeqSUYAIKKeRNSbf4bjK36n2uRkjK0BsJKI9nM3nQjg3WqT0+U8FF0yXJZqk9GhKx38ZRjYOA1OxMdiAFdVWJa7AawG0AGn1/4cgIFwBt0Wun8HCOWvcuVeAOBDXSTjMXBeDecAeNP9d1o1yQlgEoA3XBnfAfA9d3vVyCjJezyKA6pVJSMcX/Zb7r+5/BmpNjnddicDmOX+7v8E0L/a5IQzwL8RQF9hW1XJKP6z6QcsFoulDqllt4zFYrFYNFjlbrFYLHWIVe4Wi8VSh1jlbrFYLHWIVe4Wi8VSh1jlbrFYLHWIVe4Wi8VSh/w/YmzUfsIuAf8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train & verify ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('verify episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > 195:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
